Skip to content
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 72 additions & 41 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,8 @@ else
ICONV := iconv
endif

LD_OFORMAT := $(shell $(LD) --print-output-format)

INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR)

# Check code syntax with host compiler
Expand All @@ -314,14 +316,14 @@ CHECK_WARNINGS += -Werror=implicit-int -Werror=implicit-function-declaration -We

# The `cpp` command behaves differently on macOS (it behaves as if
# `-traditional-cpp` was passed) so we use `gcc -E` instead.
CPP := gcc -E
MKLDSCRIPT := tools/mkldscript
MKDMADATA := tools/mkdmadata
ELF2ROM := tools/elf2rom
BIN2C := tools/bin2c
N64TEXCONV := tools/assets/n64texconv/n64texconv
FADO := tools/fado/fado.elf
PYTHON ?= $(VENV)/bin/python3
CPP := gcc -E
MKLDSCRIPT := tools/mkldscript
MKSPECRULES := tools/mkspecrules
MKDMADATA := tools/mkdmadata
BIN2C := tools/bin2c
N64TEXCONV := tools/assets/n64texconv/n64texconv
FADO := tools/fado/fado.elf
PYTHON ?= $(VENV)/bin/python3

# Command to replace $(BUILD_DIR) in some files with the build path.
# We can't use the C preprocessor for this because it won't substitute inside string literals.
Expand Down Expand Up @@ -491,17 +493,6 @@ ASSET_FILES_BIN_COMMITTED := $(foreach dir,$(ASSET_BIN_DIRS_COMMITTED),$(wildcar
ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_BIN_EXTRACTED:.bin=.bin.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \
$(foreach f,$(ASSET_FILES_BIN_COMMITTED:.bin=.bin.inc.c),$(BUILD_DIR)/$f)

# Find all .o files included in the spec
SPEC_O_FILES := $(shell $(CPP) $(CPPFLAGS) -I. $(SPEC) | $(BUILD_DIR_REPLACE) | sed -n -E 's/^[ \t]*include[ \t]*"([a-zA-Z0-9/_.-]+\.o)"/\1/p')

# Split out reloc files
O_FILES := $(filter-out %_reloc.o,$(SPEC_O_FILES))
OVL_RELOC_FILES := $(filter %_reloc.o,$(SPEC_O_FILES))

# Automatic dependency files
# (Only asm_processor dependencies and reloc dependencies are handled for now)
DEP_FILES := $(O_FILES:.o=.d) $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d) $(BUILD_DIR)/spec.d

TEXTURE_FILES_PNG_EXTRACTED := $(foreach dir,$(ASSET_BIN_DIRS_EXTRACTED),$(wildcard $(dir)/*.png))
TEXTURE_FILES_PNG_COMMITTED := $(foreach dir,$(ASSET_BIN_DIRS_COMMITTED),$(wildcard $(dir)/*.png))
TEXTURE_FILES_JPG_EXTRACTED := $(foreach dir,$(ASSET_BIN_DIRS_EXTRACTED),$(wildcard $(dir)/*.jpg))
Expand All @@ -511,10 +502,13 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG_EXTRACTED:.png=.inc.c),$(f:
$(foreach f,$(TEXTURE_FILES_JPG_EXTRACTED:.jpg=.jpg.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \
$(foreach f,$(TEXTURE_FILES_JPG_COMMITTED:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f)

SEGMENTS_DIR := $(BUILD_DIR)/segments

# create build directories
$(shell mkdir -p $(BUILD_DIR)/baserom \
$(BUILD_DIR)/assets/text \
$(BUILD_DIR)/linker_scripts)
$(BUILD_DIR)/linker_scripts \
$(SEGMENTS_DIR))
$(shell mkdir -p $(foreach dir, \
$(SRC_DIRS) \
$(UNDECOMPILED_DATA_DIRS) \
Expand All @@ -534,6 +528,40 @@ $(shell mkdir -p $(foreach dir, \
$(dir:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)))
endif

COM_PLUGIN := tools/com-plugin/common-plugin.so
COM_PLUGIN_FLAGS =
ifeq ($(PLATFORM),IQUE)
ifeq ($(NON_MATCHING),0)
$(SEGMENTS_DIR)/boot.plf: $(BASEROM_DIR)/bss-order-boot.txt
$(SEGMENTS_DIR)/boot.plf: COM_PLUGIN_FLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order-boot.txt -plugin-opt min_align=0x10

$(SEGMENTS_DIR)/code.plf: $(BASEROM_DIR)/bss-order-code.txt
$(SEGMENTS_DIR)/code.plf: COM_PLUGIN_FLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order-code.txt -plugin-opt min_align=0x10
endif
endif

# Generate and include segment makefile rules for combining .o files into single .plf files for an entire spec segment.
# Overlay relocations will be generated from these if the spec segment has the OVERLAY flag.
# If this makefile doesn't exist or if the spec has been modified since make was last ran it will use the rule
# later on in the file to regenerate this file before including it. The test against MAKECMDGOALS ensures this
# doesn't happen if we're not running a task that needs these partially linked files; this is especially important
# for setup since the rule to generate the segment makefile rules requires setup to have ran first.
SEG_LDFLAGS = -r $(COM_PLUGIN_FLAGS) -T $(@:.plf=.ld) -Map $(@:.plf=.map)
SEG_VERBOSE = @
ifeq ($(MAKECMDGOALS),$(filter-out clean assetclean distclean setup,$(MAKECMDGOALS)))
include $(SEGMENTS_DIR)/Makefile
else
SEGMENT_FILES :=
OVL_SEGMENT_FILES :=
endif
Comment on lines +543 to +556
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bit dense.
So we are now autogenerating a Makefile for those plfs?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, the way it works is:

  • The makefile reaches this include statement (when a rule is run other than those that are mentioned in the filter-out)
  • The makefile checks to see if $(SEGMENTS_DIR)/Makefile exists.
  • Since the makefile contains a rule to build $(SEGMENTS_DIR)/Makefile, if it doesn't exist it will execute that rule and any rules it relies on to make $(SEGMENTS_DIR)/Makefile available.
  • It then includes it and carries on as if it were always there

Generating a makefile from the spec for each plf is the most convenient way to handle dependencies that I can think of that doesn't involve separating by directory (since we can't do that for multiversion anyway when files move between boot and code)

OVL_RELOC_FILES := $(OVL_SEGMENT_FILES:.plf=.reloc.o)

O_FILES := $(shell $(CPP) $(CPPFLAGS) -I. $(SPEC) | $(BUILD_DIR_REPLACE) | sed -n -E 's/^[ \t]*include[ \t]*"([a-zA-Z0-9/_.-]+\.o)"/\1/p')
MAKEROM_O_FILES := $(BUILD_DIR)/src/makerom/rom_header.o $(BUILD_DIR)/src/makerom/ipl3.o $(BUILD_DIR)/src/makerom/entry.o

# Automatic dependency files
DEP_FILES := $(O_FILES:.o=.d) $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d) $(BUILD_DIR)/spec.d $(MAKEROM_O_FILES:.o=.d)

$(BUILD_DIR)/src/boot/build.o: CPP_DEFINES += -DBUILD_CREATOR="\"$(BUILD_CREATOR)\"" -DBUILD_DATE="\"$(BUILD_DATE)\"" -DBUILD_TIME="\"$(BUILD_TIME)\""

$(BUILD_DIR)/src/audio/internal/seqplayer.o: CPP_DEFINES += -DMML_VERSION=MML_VERSION_OOT
Expand Down Expand Up @@ -742,7 +770,7 @@ else
$(BUILD_DIR)/assets/%.o: CFLAGS += -fno-zero-initialized-in-bss -fno-toplevel-reorder
$(BUILD_DIR)/src/%.o: CFLAGS += -fexec-charset=euc-jp
$(BUILD_DIR)/src/libultra/libc/ll.o: OPTFLAGS := -Ofast
$(BUILD_DIR)/src/overlays/%.o: CFLAGS += -fno-merge-constants -mno-explicit-relocs -mno-split-addresses
$(BUILD_DIR)/src/overlays/%.o: CFLAGS += -mno-explicit-relocs -mno-split-addresses
endif

#### Main Targets ###
Expand Down Expand Up @@ -821,23 +849,18 @@ else
endif

$(ROM): $(ELF)
$(ELF2ROM) -cic $(CIC) $< $@
# Here we extract the value of the _RomSize symbol to know to what size the ROM should be padded to
$(OBJCOPY) --pad-to 0x$$($(OBJDUMP) -t $< | grep _RomSize | cut -d ' ' -f 1) -O binary $< $@
$(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@
Comment on lines +853 to +854
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

woooo, happy to see we are finally ditching elf2rom.
btw, that objdump invocation seems a bit convoluted, it may be nice to have a comment about why and what it is doing.


$(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) $(COMPRESS_ARGS)
$(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@

COM_PLUGIN := tools/com-plugin/common-plugin.so
LDFLAGS := -T $(LDSCRIPT) -T $(BUILD_DIR)/linker_scripts/makerom.ld -T $(BUILD_DIR)/undefined_syms.txt --emit-relocs -Map $(MAP)

LDFLAGS := -T $(LDSCRIPT) -T $(BUILD_DIR)/linker_scripts/makerom.ld -T $(BUILD_DIR)/undefined_syms.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map $(MAP)
ifeq ($(PLATFORM),IQUE)
ifeq ($(NON_MATCHING),0)
LDFLAGS += -plugin $(COM_PLUGIN) -plugin-opt order=$(BASEROM_DIR)/bss-order.txt
$(ELF): $(BASEROM_DIR)/bss-order.txt
endif
endif

$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \
$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(MAKEROM_O_FILES) \
$(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \
$(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \
$(BUILD_DIR)/assets/audio/sequence_font_table.o $(BUILD_DIR)/assets/audio/audiobank_padding.o
$(LD) $(LDFLAGS) -o $@
Expand All @@ -861,13 +884,26 @@ $(BUILD_DIR)/spec: $(SPEC) $(SPEC_INCLUDES)
$(CPP) $(CPPFLAGS) -MD -MP -MF $@.d -MT $@ -I. $< | $(BUILD_DIR_REPLACE) > $@

$(LDSCRIPT): $(BUILD_DIR)/spec
$(MKLDSCRIPT) $< $@
$(MKLDSCRIPT) $< $@ $(BUILD_DIR)/src/makerom $(SEGMENTS_DIR)

# Generates a makefile containing rules for building .plf files
# from overlay .o files for every overlay defined in the spec.
$(SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/spec
$(MKSPECRULES) $< $(SEGMENTS_DIR) $@

# Generates relocations for each overlay after partial linking so that the final
# link step cannot later insert padding between individual overlay files after
# relocations have already been calculated.
$(SEGMENTS_DIR)/%.reloc.o: $(SEGMENTS_DIR)/%.plf
$(FADO) $< -n $(notdir $*) -o $(@:.o=.s)
$(POSTPROCESS_OBJ) $(@:.o=.s)
$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
Comment on lines +894 to +900
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So now we just pass a single plf file to fado each time we generate the relocs file?
Nice that simplify stuff a lot.


$(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt
$(CPP) $(CPPFLAGS) $< > $@

$(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
$(OBJCOPY) -I binary -O elf32-big $< $@
$(OBJCOPY) -I binary -O $(LD_OFORMAT) $< $@

$(BUILD_DIR)/data/%.o: data/%.s
$(CPP) $(CPPFLAGS) -MD -MP -MF $(@:.o=.d) -MT $@ -Iinclude $< | $(AS) $(ASFLAGS) -o $@
Expand Down Expand Up @@ -922,7 +958,7 @@ endif
$(OBJDUMP_CMD)

$(BUILD_DIR)/src/makerom/ipl3.o: $(EXTRACTED_DIR)/incbin/ipl3
$(OBJCOPY) -I binary -O elf32-big --rename-section .data=.text $< $@
$(OBJCOPY) -I binary -O $(LD_OFORMAT) --rename-section .data=.text $< $@

$(BUILD_DIR)/src/%.o: src/%.s
ifeq ($(COMPILER),ido)
Expand Down Expand Up @@ -970,15 +1006,10 @@ $(BUILD_DIR)/src/audio/game/session_init.o: src/audio/game/session_init.c $(BUIL

ifeq ($(PLATFORM),IQUE)
ifneq ($(NON_MATCHING),1)
$(BUILD_DIR)/src/overlays/misc/ovl_kaleido_scope/ovl_kaleido_scope_reloc.o: POSTPROCESS_OBJ := $(PYTHON) tools/patch_ique_kaleido_reloc.py
$(BUILD_DIR)/segments/ovl_kaleido_scope.reloc.o: POSTPROCESS_OBJ := $(PYTHON) tools/patch_ique_kaleido_reloc.py
endif
endif

$(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/spec
$(FADO) $$(tools/reloc_prereq $< $(notdir $*)) -n $(notdir $*) -o $(@:.o=.s) -M $(@:.o=.d)
$(POSTPROCESS_OBJ) $(@:.o=.s)
$(AS) $(ASFLAGS) $(@:.o=.s) -o $@

# Assets from assets/

$(BUILD_DIR)/assets/%.inc.c: assets/%.png
Expand Down
59 changes: 59 additions & 0 deletions baseroms/ique-cn/bss-order-boot.txt
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It isn't possible to just have all those symbols in the same file?
Does the linker when trying to link symbols not present on the input objects?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The BSS order is now enforced at partial link rather than final link. The plugin errors if it can't find a mentioned symbol in any of the input files, since this can often point to a user error.

Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Specification for linker plugin COMMON symbol order for the ique-cn version boot segment

build/ique-cn/boot_bss_1.o {
bk;
__osBaseCounter;
__osBbRCountWraps;
__osBbLastRCount;
__osViIntrCount;
insize;
outcnt;
bb;
__osCurrentTime;
hufts;
__osBbLastVCount;
__osTimerCounter;
__osBbVCountWraps;
__osFinalrom;
inptr;
ifd;
}

build/ique-cn/boot_bss_boot_main.o {
sBootThreadInfo;
sIdleThread;
sIdleThreadStack;
sIdleThreadInfo;
sBootThreadStack;
}

build/ique-cn/boot_bss_idle.o {
sMainThread;
sMainStack;
sMainStackInfo;
sPiMgrCmdBuff;
}

build/ique-cn/boot_bss_viconfig.o {
gViConfigMode;
gViConfigModeType;
}

build/ique-cn/boot_bss_z_std_dma.o {
sDmaMgrStackInfo;
sDmaMgrMsgQueue;
sDmaMgrMsgBuf;
sDmaMgrThread;
sDmaMgrStack;
}

build/ique-cn/boot_bss_2.o {
__osThreadSave;
__Dom2SpeedParam;
__CartRomHandle;
__osPiAccessQueue;
__Dom1SpeedParam;
gPiMgrCmdQueue;
__osBaseTimer;
__osEventStateTab;
}
Original file line number Diff line number Diff line change
@@ -1,64 +1,4 @@
// Specification for linker plugin COMMON symbol order for the ique-cn version

build/ique-cn/boot_bss_1.o {
bk;
__osBaseCounter;
__osBbRCountWraps;
__osBbLastRCount;
__osViIntrCount;
insize;
outcnt;
bb;
__osCurrentTime;
hufts;
__osBbLastVCount;
__osTimerCounter;
__osBbVCountWraps;
__osFinalrom;
inptr;
ifd;
}

build/ique-cn/boot_bss_boot_main.o {
sBootThreadInfo;
sIdleThread;
sIdleThreadStack;
sIdleThreadInfo;
sBootThreadStack;
}

build/ique-cn/boot_bss_idle.o {
sMainThread;
sMainStack;
sMainStackInfo;
sPiMgrCmdBuff;
}

build/ique-cn/boot_bss_viconfig.o {
gViConfigMode;
gViConfigModeType;
}

build/ique-cn/boot_bss_z_std_dma.o {
sDmaMgrStackInfo;
sDmaMgrMsgQueue;
sDmaMgrMsgBuf;
sDmaMgrThread;
sDmaMgrStack;
}

build/ique-cn/boot_bss_2.o {
__osThreadSave;
__Dom2SpeedParam;
__CartRomHandle;
__osPiAccessQueue;
__Dom1SpeedParam;
gPiMgrCmdQueue;
__osBaseTimer;
__osEventStateTab;
}


// Specification for linker plugin COMMON symbol order for the ique-cn version code segment

build/ique-cn/code_bss_1.o {
D_8015FA8C;
Expand Down
13 changes: 12 additions & 1 deletion first_diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ def decodeInstruction(bytesDiff: bytes, mapFile: mapfile_parser.MapFile) -> str:

return instr.disassemble(immOverride=immOverride, extraLJust=-20)

def plfResolver(x: Path) -> Path|None:
if x.suffix == ".plf":
plf_map_path = x.with_suffix(".map")
if plf_map_path.exists():
return plf_map_path
return None

def firstDiffMain():
parser = argparse.ArgumentParser(description="Find the first difference(s) between the built ROM and the base ROM.")

Expand All @@ -43,7 +50,11 @@ def firstDiffMain():
EXPECTEDROM = Path(f"baseroms/{args.oot_version}/baserom-decompressed.z64")
EXPECTEDMAP = "expected" / BUILTMAP

mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count, mismatchSize=True, addColons=args.add_colons, bytesConverterCallback=decodeInstruction)
mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count,
mismatchSize=True, addColons=args.add_colons,
bytesConverterCallback=decodeInstruction,
plfResolver=plfResolver,
plfResolverExpected=plfResolver)

if __name__ == "__main__":
firstDiffMain()
Loading