Merge pull request #69 from happyleavesaoc/HD

HD >= 4.6 support

Author: happyleavesaoc

README.md

@@ -8,6 +8,7 @@ Age of Empires II recorded game parsing and summarization in Python 3.
 - The Conquerors (`.mgx`)
 - Userpatch 1.4 (`.mgz`)
 - Userpatch 1.5 (`.mgz`)
+- HD Edition >= 4.6 (`.aoe2record`)
 - Definitive Edition (`.aoe2record`)
 
 ## Architecture
@@ -30,6 +31,8 @@ Abstractions take parser output as input and return an object with normalized da
 | The Conquerors (`.mgx`) | | ✓ | | ✓ | ✓ | |
 | Userpatch <= 1.4 (`.mgz`) | | ✓ | | ✓ | ✓ | ✓ |
 | Userpatch 1.5 (`.mgz`) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
+| HD Edition >= 4.6 | | ✓ | | ✓ | ✓ | ✓ |
+| HD Edition 5.8 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
 | Definitive Edition <= 13.34 (`.aoe2record`) | | ✓ | | ✓ | ✓ | ✓ |
 | Definitive Edition > 13.34 (`.aoe2record`) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
 

mgz/__init__.py

@@ -12,12 +12,14 @@
 from mgz.header.scenario import scenario
 from mgz.header.lobby import lobby
 from mgz.header.de import de
+from mgz.header.hd import hd
 
 
 compressed_header = Struct(
     "game_version"/CString(encoding='latin1'),
     "save_version"/VersionAdapter(Float32l),
     "version"/Computed(lambda ctx: get_version(ctx.game_version, ctx.save_version, None)),
+    "hd"/If(lambda ctx: ctx.version == Version.HD and ctx.save_version > 12.34, hd),
     "de"/If(lambda ctx: ctx.version == Version.DE, de),
     ai,
     replay,
@@ -29,6 +31,7 @@
     Terminated
 )
 
+
 subheader = Struct(
     "check"/Peek(Int32ul),
     "chapter_address"/If(lambda ctx: ctx.check < 100000000, Int32ul),

mgz/enums.py

@@ -203,7 +203,8 @@ def ResourceLevelEnum(ctx):
         medium=2,
         high=3,
         unknown1=4,
-        unknown2=5
+        unknown2=5,
+        unknown3=6
     )
 
 def RevealMapEnum(ctx):

mgz/fast/__init__.py

@@ -37,6 +37,7 @@ class Action(Enum):
     GROUP_MULTI_WAYPOINTS = 31
     CHAPTER = 32
     DE_ATTACK_MOVE = 33
+    HD_UNKNOWN_34 = 34
     DE_UNKNOWN_35 = 35
     DE_UNKNOWN_37 = 37
     DE_AUTOSCOUT = 38

mgz/fast/header.py

@@ -44,11 +44,18 @@ def aoc_string(data):
 
 def de_string(data):
     """Read DE string."""
-    assert data.read(2) == b'\x60\x0A'
+    assert data.read(2) == b'\x60\x0a'
     length = unpack('<h', data)
     return unpack(f'<{length}s', data)
 
 
+def hd_string(data):
+    """Read HD string."""
+    length = unpack('<h', data)
+    assert data.read(2) == b'\x60\x0a'
+    return unpack(f'<{length}s', data)
+
+
 def parse_object(data, offset):
     """Parse an object."""
     class_id, object_id, instance_id, pos_x, pos_y = struct.unpack_from('<bxH14xIxff', data, offset)
@@ -141,11 +148,13 @@ def parse_lobby(data, version, save):
         if save >= 20.06:
             data.read(9)
     data.read(8)
-    if version is not Version.DE:
+    if version not in (Version.DE, Version.HD):
         data.read(1)
     reveal_map_id, map_size, population, game_type_id, lock_teams = unpack('I4xIIbb', data)
-    if version is Version.DE:
-        data.read(9)
+    if version in (Version.DE, Version.HD):
+        data.read(5)
+        if save >= 13.13:
+            data.read(4)
     chat = []
     for _ in range(0, unpack('<I', data)):
         message = data.read(unpack('<I', data)).strip(b'\x00')
@@ -157,7 +166,7 @@ def parse_lobby(data, version, save):
     return dict(
         reveal_map_id=reveal_map_id,
         map_size=map_size,
-        population=population * (25 if version is not Version.DE else 1),
+        population=population * (25 if version not in (Version.DE, Version.HD) else 1),
         game_type_id=game_type_id,
         lock_teams=lock_teams == 1,
         chat=chat,
@@ -175,7 +184,7 @@ def parse_map(data, version):
     size_x, size_y, zone_num = unpack('<III', data)
     tile_num = size_x * size_y
     for _ in range(zone_num):
-        if version is Version.DE:
+        if version in (Version.DE, Version.HD):
             data.read(2048 + (tile_num * 2))
         else:
             data.read(1275 + tile_num)
@@ -216,15 +225,19 @@ def parse_scenario(data, num_players, version):
     data.read(196)
     for _ in range(0, 16):
         data.read(24)
-        if version is Version.DE:
+        if version in (Version.DE, Version.HD):
             data.read(4)
     data.read(12672)
     if version is Version.DE:
         data.read(196)
     else:
         for _ in range(0, 16):
             data.read(332)
+    if version is Version.HD:
+        data.read(644)
     data.read(88)
+    if version is Version.HD:
+        data.read(16)
     map_id, difficulty_id = unpack('<II', data)
     remainder = data.read()
     if version is Version.DE:
@@ -293,7 +306,60 @@ def parse_de(data, version, save):
         players=players,
         guid=str(uuid.UUID(bytes=guid)),
         lobby=lobby.decode('utf-8'),
-        mod=mod
+        mod=mod.decode('utf-8')
+    )
+
+
+def parse_hd(data, version, save):
+    """Parse HD-specifc header."""
+    if version is not Version.HD or save <= 12.34:
+        return None
+    data.read(12)
+    dlc_count = unpack('<I', data)
+    data.read(dlc_count * 4)
+    data.read(8)
+    map_id = unpack('<I', data)
+    data.read(80)
+    players = []
+    for _ in range(8):
+        data.read(4)
+        color_id = unpack('<i', data)
+        data.read(12)
+        civilization_id = unpack('<I', data)
+        hd_string(data)
+        data.read(1)
+        hd_string(data)
+        name = hd_string(data)
+        data.read(4)
+        steam_id, number = unpack('<Qi', data)
+        data.read(8)
+        if name:
+            players.append(dict(
+                number=number,
+                color_id=color_id,
+                name=name,
+                profile_id=steam_id,
+                civilization_id=civilization_id
+            ))
+    data.read(26)
+    hd_string(data)
+    data.read(8)
+    hd_string(data)
+    data.read(8)
+    hd_string(data)
+    data.read(8)
+    guid = data.read(16)
+    lobby = hd_string(data)
+    mod = hd_string(data)
+    data.read(8)
+    hd_string(data)
+    data.read(4)
+    return dict(
+        players=players,
+        guid=str(uuid.UUID(bytes=guid)),
+        lobby=lobby.decode('utf-8'),
+        mod=mod.decode('utf-8'),
+        map_id=map_id
     )
 
 
@@ -310,15 +376,15 @@ def parse_version(header, data):
     log = unpack('<I', data)
     game, save = unpack('<7sxf', header)
     version = get_version(game.decode('ascii'), round(save, 2), log)
-    if version not in (Version.USERPATCH15, Version.DE):
+    if version not in (Version.USERPATCH15, Version.DE, Version.HD):
         raise RuntimeError(f"{version} not supported")
     return version, round(save, 2)
 
 
 def parse_players(header, num_players, version):
     """Parse all players."""
     cur = header.tell()
-    gaia = b'Gaia' if version is Version.DE else b'GAIA'
+    gaia = b'Gaia' if version in (Version.DE, Version.HD) else b'GAIA'
     anchor = header.read().find(b'\x05\x00' + gaia + b'\x00')
     header.seek(cur + anchor - num_players - 43)
     mod = parse_mod(header, num_players, version)
@@ -350,6 +416,7 @@ def parse(data):
         header = decompress(data)
         version, save = parse_version(header, data)
         de = parse_de(header, version, save)
+        hd = parse_hd(header, version, save)
         metadata, num_players = parse_metadata(header)
         map_ = parse_map(header, version)
         players, mod = parse_players(header, num_players, version)
@@ -362,6 +429,7 @@ def parse(data):
         players=players,
         map=map_,
         de=de,
+        hd=hd,
         mod=mod,
         metadata=metadata,
         scenario=scenario,

mgz/header/de.py

@@ -74,8 +74,7 @@
         "resolved_team_id"/Byte,
         "dat_crc"/Bytes(8),
         "mp_game_version"/Byte,
-        "civ_id"/Byte,
-        Const(b"\x00\x00\x00"),
+        "civ_id"/Int32ul,
         "ai_type"/de_string,
         "ai_civ_name_index"/Byte,
         "ai_name"/de_string,

mgz/header/hd.py

@@ -0,0 +1,145 @@
+from construct import (
+    Struct, Int32ul, Float32l, Array, Padding, Flag, If,
+    Byte, Int16ul, Bytes, Int32sl, Peek, Const, RepeatUntil,
+    Int64ul, Computed, Embedded, IfThenElse
+)
+
+from mgz.enums import VictoryEnum, ResourceLevelEnum, AgeEnum, PlayerTypeEnum, DifficultyEnum
+from mgz.util import find_save_version
+
+separator = Const(b"\xa3_\x02\x00")
+
+hd_string = Struct(
+    "length"/Int16ul,
+    Const(b"\x60\x0A"),
+    "value"/Bytes(lambda ctx: ctx.length)
+)
+
+test_57 = "test_57"/Struct(
+    "check"/Int32ul,
+    Padding(4),
+    If(lambda ctx: ctx._._.version >= 1006, Bytes(1)),
+    Padding(15),
+    hd_string,
+    Padding(1),
+    If(lambda ctx: ctx._._.version >= 1005, hd_string),
+    hd_string,
+    Padding(16),
+    "test"/Int32ul,
+    "is_57"/Computed(lambda ctx: ctx.check == ctx.test)
+)
+
+player = Struct(
+    "dlc_id"/Int32ul,
+    "color_id"/Int32ul,
+    "unk1_1006"/If(lambda ctx: ctx._._.version >= 1006, Bytes(1)),
+    "unk"/Bytes(2),
+    "dat_crc"/Bytes(4),
+    "mp_game_version"/Byte,
+    "team_index"/Int32ul,
+    "civ_id"/Int32ul,
+    "ai_type"/hd_string,
+    "ai_civ_name_index"/Byte,
+    "ai_name"/If(lambda ctx: ctx._._.version >= 1005, hd_string),
+    "name"/hd_string,
+    "type"/PlayerTypeEnum(Int32ul),
+    "steam_id"/Int64ul,
+    "player_number"/Int32sl,
+    Embedded(If(lambda ctx: ctx._._.version >= 1006 and not ctx._.test_57.is_57, Struct(
+        "hd_rm_rating"/Int32ul,
+        "hd_dm_rating"/Int32ul,
+    )))
+)
+
+hd = "hd"/Struct(
+    "version"/Float32l,
+    "interval_version"/Int32ul,
+    "game_options_version"/Int32ul,
+    "dlc_count"/Int32ul,
+    "dlc_ids"/Array(lambda ctx: ctx.dlc_count, Int32ul),
+    "dataset_ref"/Int32ul,
+    Peek("difficulty_id"/Int32ul),
+    DifficultyEnum("difficulty"/Int32ul),
+    "selected_map_id"/Int32ul,
+    "resolved_map_id"/Int32ul,
+    "reveal_map"/Int32ul,
+    Peek("victory_type_id"/Int32ul),
+    VictoryEnum("victory_type"/Int32ul),
+    Peek("starting_resources_id"/Int32ul),
+    ResourceLevelEnum("starting_resources"/Int32ul),
+    "starting_age_id"/Int32ul,
+    "starting_age"/AgeEnum(Computed(lambda ctx: ctx.starting_age_id)),
+    "ending_age_id"/Int32ul,
+    "ending_age"/AgeEnum(Computed(lambda ctx: ctx.ending_age_id)),
+    "game_type"/If(lambda ctx: ctx.version >= 1006, Int32ul),
+    separator,
+    "ver1000"/If(lambda ctx: ctx.version == 1000, Struct(
+        "map_name"/hd_string,
+        "unk"/hd_string
+    )),
+    separator,
+    "speed"/Float32l,
+    "treaty_length"/Int32ul,
+    "population_limit"/Int32ul,
+    "num_players"/Int32ul,
+    "unused_player_color"/Int32ul,
+    "victory_amount"/Int32ul,
+    separator,
+    "trade_enabled"/Flag,
+    "team_bonus_disabled"/Flag,
+    "random_positions"/Flag,
+    "all_techs"/Flag,
+    "num_starting_units"/Byte,
+    "lock_teams"/Flag,
+    "lock_speed"/Flag,
+    "multiplayer"/Flag,
+    "cheats"/Flag,
+    "record_game"/Flag,
+    "animals_enabled"/Flag,
+    "predators_enabled"/Flag,
+    "turbo_enabled"/Flag,
+    "shared_exploration"/Flag,
+    "team_positions"/Flag,
+    "unk"/Bytes(1),
+    Embedded(IfThenElse(lambda ctx: ctx.version == 1000,
+        Struct(
+            Bytes(40*3),
+            separator,
+            Bytes(40),
+            "strings"/Array(8, hd_string),
+            Bytes(16),
+            separator,
+            Bytes(10),
+        ),
+        Struct(
+            Peek(test_57),
+            "players"/Array(8, player),
+            "fog_of_war"/Flag,
+            "cheat_notifications"/Flag,
+            "colored_chat"/Flag,
+            Bytes(9),
+            separator,
+            "is_ranked"/Flag,
+            "allow_specs"/Flag,
+            "lobby_visibility"/Int32ul,
+            "custom_random_map_file_crc"/Int32ul,
+            "custom_scenario_or_campaign_file"/hd_string,
+            Bytes(8),
+            "custom_random_map_file"/hd_string,
+            Bytes(8),
+            "custom_random_map_scenarion_file"/hd_string,
+            Bytes(8),
+            "guid"/Bytes(16),
+            "lobby_name"/hd_string,
+            "modded_dataset"/hd_string,
+            "modded_dataset_workshop_id"/Bytes(4),
+            If(lambda ctx: ctx._.version >= 1005,
+                Struct(
+                    Bytes(4),
+                    hd_string,
+                    Bytes(4)
+                )
+            )
+        )
+    ))
+)