support 61.5, 62.0

Author: happyleavesaoc

mgz/fast/enums.py

@@ -44,6 +44,7 @@ class Action(Enum):
     DE_UNKNOWN_40 = 40
     DE_TRANSFORM = 41
     RATHA_ABILITY = 43
+    DE_107_A = 44
     AI_COMMAND = 53
     DE_UNKNOWN_80 = 80
     MAKE = 100
@@ -74,6 +75,7 @@ class Action(Enum):
     DE_UNKNOWN_135 = 135
     DE_UNKNOWN_136 = 136
     DE_UNKNOWN_138 = 138
+    DE_107_B = 140
     DE_TRIBUTE = 196
     POSTGAME = 255
 

mgz/fast/header.py

@@ -110,7 +110,10 @@ def parse_mod(header, num_players, version):
 
 def parse_player(header, player_number, num_players, save):
     """Parse a player (and objects)."""
-    type_, *diplomacy, name_length = unpack(f'<bx{num_players}x9i5xh', header)
+    rep = 9
+    if save >= 61.5:
+        rep = num_players
+    type_, *diplomacy, name_length = unpack(f'<bx{num_players}x{rep}i5xh', header)
     name, resources = unpack(f'<{name_length - 1}s2xIx', header)
     header.read(resources * 4)
     start_x, start_y, civilization_id, color_id = unpack('<xff9xb3xbx', header)
@@ -133,12 +136,16 @@ def parse_player(header, player_number, num_players, save):
     if save >= 37:
         offset = header.tell()
         data = header.read(100)
+        device = data[8]
         # Jump to the end of player data
         player_end = re.search(b'\xff\xff\xff\xff\xff\xff\xff\xff.\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b', data, re.DOTALL)
         if not player_end:
-            raise RuntimeError("could not find player end")
-        device = data[8]
-        header.seek(offset + player_end.end())
+            # only a failure if this is not the last player, since we seek to the next block anyway
+            # this issue happens on restored games
+            if player_number < num_players - 1:
+                raise RuntimeError("could not find player end")
+        else:
+            header.seek(offset + player_end.end())
 
     return dict(
         number=player_number,
@@ -194,12 +201,13 @@ def parse_lobby(data, version, save):
     )
 
 
-def parse_map(data, version):
+def parse_map(data, version, save):
     """Parse map."""
-    data.read(60)
     tile_format = '<xbbx'
     if version is Version.DE:
         tile_format = '<bxb6x'
+        if save >= 62.0:
+            tile_format = '<bxxb6x'
         data.read(8)
     size_x, size_y, zone_num = unpack('<III', data)
     tile_num = size_x * size_y
@@ -220,6 +228,8 @@ def parse_map(data, version):
         data.read(num_obs * 8)
     x2, y2 = unpack('<II', data)
     data.read(x2 * y2 * 4)
+    if save >= 61.5:
+        data.read(x2 * y2 * 4)
     restore_time = unpack('<I', data)
     #if restore_time > 0:
     #    raise RuntimeError("restored matches can't be parsed yet")
@@ -233,7 +243,10 @@ def parse_map(data, version):
 
 def parse_scenario(data, num_players, version, save):
     """Parse scenario section."""
-    data.read(4455)
+    next_uid, scenario_version = unpack('<II', data)
+    if save >= 61.5:
+        data.read(72)
+    data.read(4447)
     scenario_filename = None
     if version is Version.DE:
         data.read(102)
@@ -264,7 +277,9 @@ def parse_scenario(data, num_players, version, save):
     map_id, difficulty_id = unpack('<II', data)
     remainder = data.read()
     if version is Version.DE:
-        if save >= 37:
+        if save >= 61.5:
+            settings_version = 3.6
+        elif save >= 37:
             settings_version = 3.5
         elif save >= 26.21:
             settings_version = 3.2
@@ -346,20 +361,26 @@ def parse_de(data, version, save, skip=False):
     for i in range(0, unpack('<I', data)):
         dlc_ids.append(unpack('<I', data))
     data.read(4)
-    difficulty_id = unpack('<I', data)
+    if save >= 61.5:
+        map_dimension = unpack('<I', data)
+    else:
+        difficulty_id = unpack('<I', data)
     data.read(4)
     rms_map_id = unpack('<I', data)
     data.read(4)
     victory_type_id = unpack('<I', data)
     starting_resources_id = unpack('<I', data)
     starting_age_id = unpack('<I', data)
     ending_age_id = unpack('<I', data)
-    data.read(8)
-    speed = unpack('<d', data)
+    data.read(12)
+    speed = unpack('<f', data)
     treaty_length = unpack('<I', data)
     population_limit = unpack('<I', data)
     num_players = unpack('<I', data)
     data.read(14)
+    if save >= 61.5:
+        # not sure if this is difficulty under 61.5 or not
+        difficulty_id = unpack('<B', data)
     random_positions, all_technologies = unpack('<bb', data)
     data.read(1)
     lock_teams = unpack('<b', data)
@@ -385,6 +406,8 @@ def parse_de(data, version, save, skip=False):
         team_id = unpack('<b', data)
         data.read(9)
         civilization_id = unpack('<I', data)
+        if save >= 61.5:
+            data.read(4)
         de_string(data)
         data.read(1)
         ai_name = de_string(data)
@@ -412,6 +435,8 @@ def parse_de(data, version, save, skip=False):
     data.read(12)
     if save >= 37:
         for _ in range(8 - num_players):
+            if save >= 61.5:
+                data.read(4)
             data.read(12)
             de_string(data)
             data.read(1)
@@ -463,6 +488,8 @@ def parse_de(data, version, save, skip=False):
         data.read(3)
     if save > 50:
         data.read(8)
+    if save >= 61.5:
+        data.read(1)
     if not skip:
         de_string(data)
         data.read(8)
@@ -595,11 +622,17 @@ def parse_players(header, num_players, version, save):
     cur = header.tell()
     gaia = b'Gaia' if version in (Version.DE, Version.HD) else b'GAIA'
     anchor = header.read().find(b'\x05\x00' + gaia + b'\x00')
-    header.seek(cur + anchor - num_players - 43)
+    rev = 43
+    if save >= 61.5:
+        rev = 7 + (num_players * 4)
+    header.seek(cur + anchor - num_players - rev)
     mod = parse_mod(header, num_players, version)
     players = [parse_player(header, number, num_players, save) for number in range(num_players)]
     cur = header.tell()
-    points_version = header.read().find(b'\x00\x00\x00@')
+    pv = b'\x00\x00\x00@'
+    if save >= 61.5:
+        pv = b'\x66\x66\x06\x40'
+    points_version = header.read().find(pv)
     header.seek(cur)
     header.read(points_version)
     for _ in range(num_players):
@@ -611,7 +644,7 @@ def parse_players(header, num_players, version, save):
     return [p[0] for p in players], mod, players[0][1]
 
 
-def parse_metadata(header, skip_ai=True):
+def parse_metadata(header, save, skip_ai=True):
     """Parse recorded game metadata."""
     ai = unpack('<I', header)
 
@@ -630,6 +663,12 @@ def parse_metadata(header, skip_ai=True):
         header.seek(offset + ai_end.end())
 
     game_speed, owner_id, num_players, cheats = unpack('<24xf17xhbxb', header)
+
+    if save < 61.5:
+        header.read(60)
+    else:
+        header.read(24 + (num_players * 4))
+
     return dict(
         speed=game_speed,
         owner_id=owner_id,
@@ -646,8 +685,8 @@ def parse(data):
             raise RuntimeError(f"{version} not supported")
         de = parse_de(header, version, save)
         hd = parse_hd(header, version, save)
-        metadata, num_players = parse_metadata(header)
-        map_ = parse_map(header, version)
+        metadata, num_players = parse_metadata(header, save)
+        map_ = parse_map(header, version, save)
         players, mod, device = parse_players(header, num_players, version, save)
         scenario = parse_scenario(header, num_players, version, save)
         lobby = parse_lobby(header, version, save)

mgz/header/de.py

@@ -28,6 +28,7 @@
     "dat_crc"/Bytes(8),
     "mp_game_version"/Byte,
     "civ_id"/Int32ul,
+    "unk"/If(lambda ctx: find_save_version(ctx) >= 61.5, Int32ul),
     "ai_type"/de_string,
     "ai_civ_name_index"/Byte,
     "ai_name"/de_string,
@@ -59,8 +60,8 @@
     "dlc_count"/Int32ul,
     "dlc_ids"/Array(lambda ctx: ctx.dlc_count, Int32ul),
     "dataset_ref"/Int32ul,
-    Peek("difficulty_id"/Int32ul),
-    DifficultyEnum("difficulty"/Int32ul),
+    Peek("difficulty_id"/Int32ul), # map size now?
+    "diff"/Int32ul,
     "selected_map_id"/Int32ul,
     "resolved_map_id"/Int32ul,
     "reveal_map"/Int32ul,
@@ -81,6 +82,7 @@
     "num_players"/Int32ul,
     "unused_player_color"/Int32ul,
     "victory_amount"/Int32sl,
+    "unk_byte"/If(lambda ctx: find_save_version(ctx) >= 61.5, Flag),
     separator,
     "trade_enabled"/Flag,
     "team_bonus_disabled"/Flag,
@@ -108,6 +110,7 @@
     "cheat_notifications"/Flag,
     "colored_chat"/Flag,
     "empty_slots"/If(lambda ctx: find_save_version(ctx) >= 37, Array(lambda ctx: 8 - ctx.num_players, Struct(
+        "unk"/If(lambda ctx: find_save_version(ctx) >= 61.5, Int32ul),
         "i0x"/Int32ul,
         "i0a"/Int32ul,
         "i0b"/Int32ul,
@@ -154,6 +157,7 @@
     If(lambda ctx: find_save_version(ctx) >= 26.16, Bytes(8)),
     If(lambda ctx: find_save_version(ctx) >= 37, Bytes(3)),
     If(lambda ctx: find_save_version(ctx) >= 50, Bytes(8)),
+    If(lambda ctx: find_save_version(ctx) >= 61.5, Flag),
     de_string,
     Bytes(5),
     If(lambda ctx: find_save_version(ctx) >= 13.13, Byte),

mgz/header/initial.py

@@ -8,12 +8,12 @@
 from mgz.enums import MyDiplomacyEnum, TheirDiplomacyEnum
 from mgz.header.objects import existing_object
 from mgz.header.playerstats import player_stats
-from mgz.util import Find, GotoObjectsEnd, RepeatUpTo, Version
+from mgz.util import Find, GotoObjectsEnd, RepeatUpTo, Version, find_save_version
 
 # Player attributes.
 attributes = "attributes"/Struct(
     Array(lambda ctx: ctx._._._.replay.num_players, TheirDiplomacyEnum("their_diplomacy"/Byte)),
-    Array(9, MyDiplomacyEnum("my_diplomacy"/Int32sl)),
+    Array(lambda ctx: ctx._._._.replay.num_players if find_save_version(ctx) >= 61.5 else 9, MyDiplomacyEnum("my_diplomacy"/Int32sl)),
     "allied_los"/Int32ul,
     "allied_victory"/Flag,
     "player_name_length"/Int16ul,

mgz/header/map_info.py

@@ -3,7 +3,7 @@
 from construct import (Array, Byte, Computed, Embedded, Flag, IfThenElse,
                        Int32ul, Padding, Struct, Int16sl, If, Peek)
 
-from mgz.util import Version
+from mgz.util import Version, find_save_version
 
 # pylint: disable=invalid-name, bad-continuation
 
@@ -13,6 +13,7 @@
     "terrain_type"/Byte,
     Embedded(IfThenElse(lambda ctx: ctx._._.version == Version.DE,
         Embedded(Struct(
+            If(lambda ctx: ctx._._._.save_version >= 62.0, Byte),
             Padding(1), # copy of previous byte
             "elevation"/Byte,
             "unk0"/Int16sl,
@@ -59,4 +60,5 @@
     "size_x_2"/Int32ul,
     "size_y_2"/Int32ul,
     Padding(lambda ctx: ctx.tile_num * 4), # visibility
+    If(lambda ctx: find_save_version(ctx) >= 61.5, Padding(lambda ctx: ctx.tile_num * 4))
 )

mgz/header/objects.py

@@ -112,7 +112,9 @@
             If(lambda ctx: find_save_version(ctx) >= 25.22 and find_type(ctx) == 10, Bytes(1)),
             If(lambda ctx: find_save_version(ctx) < 25.22 and find_type(ctx) == 10 and ctx.peek[0] == 0 and ctx.peek[0:2] != b"\x00\x0b", Bytes(1)),
             If(lambda ctx: find_type(ctx) == 20 and ctx.peek[4] == 0 and ctx.peek[4:6] != b"\x00\x0b", Bytes(1)),
-        ))
+        )),
+        If(lambda ctx: find_save_version(ctx) >= 61.5, Bytes(14)),
+        If(lambda ctx: find_save_version(ctx) >= 62.0, Bytes(4))
     )),
     "hd_extension"/If(lambda ctx: find_version(ctx) == Version.HD and find_save_version(ctx) > 12.36, Struct(
         "flag"/Flag,
@@ -397,6 +399,7 @@
     "de"/If(lambda ctx: find_version(ctx) == Version.DE, Bytes(14)),
     "de_2"/If(lambda ctx: find_save_version(ctx) >= 26.16, Bytes(16)),
     "de_3"/If(lambda ctx: find_save_version(ctx) >= 26.18, Bytes(1)),
+    "de_4"/If(lambda ctx: find_save_version(ctx) >= 61.5, Bytes(4)),
     "next_volley"/Byte,
     "using_special_animation"/Byte,
     "own_base"/Byte,
@@ -429,6 +432,7 @@
     "de_unknown3"/If(lambda ctx: 26.18 > find_save_version(ctx) >= 26.16, Bytes(5)),
     "de_unknown4"/If(lambda ctx: find_save_version(ctx) >= 26.18, Bytes(4)),
     "de_unknown5"/If(lambda ctx: find_save_version(ctx) >= 50, Bytes(48)),
+    "de_unknown6"/If(lambda ctx: find_save_version(ctx) >= 61.5, Bytes(44))
 )
 
 production_queue = "production_queue"/Struct(
@@ -474,6 +478,7 @@
     "de_unk_3"/If(lambda ctx: find_save_version(ctx) >= 25.22, Byte),
     "de_unk_4"/If(lambda ctx: find_save_version(ctx) >= 26.16, Bytes(4)),
     "de_unk_5"/If(lambda ctx: find_save_version(ctx) >= 50.4, Bytes(4)),
+    "de_unk_6"/If(lambda ctx: find_save_version(ctx) >= 61.5, Bytes(12)),
 )
 
 

mgz/header/replay.py

@@ -2,7 +2,7 @@
 
 from construct import Array, Byte, Flag, Float32l, Int16ul, Int32sl, Int32ul, Padding, Struct, If, Embedded
 
-from mgz.util import Version
+from mgz.util import Version, find_save_version
 
 # pylint: disable=invalid-name
 
@@ -35,6 +35,6 @@
     "king_campaign_player"/Byte,
     "king_campaign_scenario"/Byte,
     "player_turn"/Int32ul,
-    "player_time_delta"/Array(9, "turn"/Int32ul),
+    "player_time_delta"/Array(lambda obj: obj.num_players if find_save_version(obj) >= 61.5 else 9, "turn"/Int32ul),
     If(lambda ctx: ctx._.version == Version.DE, Padding(8))
 )

mgz/header/scenario.py

@@ -14,8 +14,10 @@
 scenario_header = "scenario_header"/Struct(
     "next_uid"/Int32ul,
     "scenario_version"/Float32l,
+    If(lambda ctx: find_save_version(ctx) >= 61.5, Padding(8)),
     Array(16, "names"/String(256)),
     Array(16, "player_ids"/Int32ul),
+    If(lambda ctx: find_save_version(ctx) >= 61.5, Padding(64)),
     Array(16, "player_data"/Struct(
         "active"/Int32ul,
         "human"/Int32ul,
@@ -147,7 +149,8 @@
             If(lambda ctx: 26.16 > find_save_version(ctx) >= 25.22, Find(struct.pack('<d', 2.6), None)),
             If(lambda ctx: 26.21 > find_save_version(ctx) >= 26.16, Find(struct.pack('<d', 3.0), None)),
             If(lambda ctx: 37 > find_save_version(ctx) >= 26.21, Find(struct.pack('<d', 3.2), None)),
-            If(lambda ctx: find_save_version(ctx) >= 37, Find(struct.pack('<d', 3.5), None))
+            If(lambda ctx: 61.5 > find_save_version(ctx) >= 37, Find(struct.pack('<d', 3.5), None)),
+            If(lambda ctx: find_save_version(ctx) >= 61.5, Find(struct.pack('<d', 3.6), None))
         ),
         "end_of_game_settings"/Find(b'\x9a\x99\x99\x99\x99\x99\xf9\\x3f', None)
     )

mgz/model/__init__.py

@@ -119,8 +119,8 @@ def parse_match(handle):
             data['map']['tiles'],
             de_seed=data['lobby']['seed']
         )
-    except ValueError:
-        raise RuntimeError("could not get map data")
+    except ValueError as e:
+        raise RuntimeError(f"could not get map data: {e}")
 
     # Handle DE-specific data
     rated = None

mgz/util.py

@@ -271,7 +271,10 @@ def _parse(self, stream, context, path):
                 marker -= 1
                 count += 1
             # Backtrack through the rest of the next player structure
-            backtrack = 43 + num_players
+            offset = 9 * 4
+            if save_version >= 61.5:
+                offset = num_players * 4
+            backtrack = 7 + num_players + offset
         # Otherwise, this is the last player
         else:
             # Search for the scenario header