map export to geoparquet using GDAL

.github/workflows/generate-symbols.yml

@@ -36,6 +36,7 @@ on:
         default: default
         options:
         - default
+        - experimental
         - testing
         - adventure_test
         - beta

.github/workflows/watch-df-release.yml

@@ -15,13 +15,17 @@ jobs:
       fail-fast: false
       matrix:
         # df_steam_branch: which DF Steam branch to watch
-        # platform: leave blank to default to all
+        # platform: for symbols generation; leave blank to default to all
         # structures_ref: leave blank to default to master
         # dfhack_ref: leave blank if no structures update is desired
         # steam_branch: leave blank if no DFHack steam push is desired
         include:
           - df_steam_branch: public
           - df_steam_branch: beta
+          - df_steam_branch: experimental
+            structures_ref: experimental
+            dfhack_ref: experimental
+            steam_branch: experimental
     steps:
     - name: Fetch state
       uses: actions/cache/restore@v4

CMakeLists.txt

@@ -6,9 +6,9 @@ cmake_policy(SET CMP0048 NEW)
 cmake_policy(SET CMP0074 NEW)
 
 # set up versioning.
-set(DF_VERSION "51.05")
-set(DFHACK_RELEASE "r1")
-set(DFHACK_PRERELEASE FALSE)
+set(DF_VERSION "51.06")
+set(DFHACK_RELEASE "r2rc1")
+set(DFHACK_PRERELEASE TRUE)
 
 set(DFHACK_VERSION "${DF_VERSION}-${DFHACK_RELEASE}")
 set(DFHACK_ABI_VERSION 2)
@@ -451,7 +451,6 @@ if(INSTALL_DATA_FILES)
     install(FILES LICENSE.rst DESTINATION ${DFHACK_USERDOC_DESTINATION})
     install(FILES docs/changelog-placeholder.txt DESTINATION ${DFHACK_USERDOC_DESTINATION} RENAME changelog.txt)
     install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/depends/luacov/src/luacov/ DESTINATION ${DFHACK_DATA_DESTINATION}/lua/luacov)
-    install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/patches/ DESTINATION ${DFHACK_DATA_DESTINATION}/patches)
 endif()
 
 if(INSTALL_SCRIPTS)

data/CMakeLists.txt

@@ -24,6 +24,10 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/blueprints/
         FILES_MATCHING PATTERN "*"
                        PATTERN blueprints/test EXCLUDE)
 
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/patches/
+    DESTINATION ${DFHACK_DATA_DESTINATION}/patches
+)
+
 if(BUILD_TESTS)
     install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/blueprints/test/
             DESTINATION "${DFHACK_DATA_DESTINATION}/data/blueprints/test")

data/orders/basic.json

@@ -114,27 +114,6 @@
 		"job" : "CustomReaction",
 		"reaction" : "BREW_DRINK_FROM_PLANT_GROWTH"
 	},
-	{
-		"amount_left" : 1,
-		"amount_total" : 1,
-		"frequency" : "Daily",
-		"id" : 3,
-		"is_active" : false,
-		"is_validated" : false,
-		"item_conditions" : 
-		[
-			{
-				"condition" : "AtLeast",
-				"flags" : 
-				[
-					"unrotten",
-					"milk"
-				],
-				"value" : 2
-			}
-		],
-		"job" : "MakeCheese"
-	},
 	{
 		"amount_left" : 1,
 		"amount_total" : 1,

data/patches/README.md

@@ -0,0 +1,13 @@
+Place IDA-exported `.dif` files for use by `binpatch` in subdirectories of this
+directory. Each `.dif` file must be in a subdirectory named after the full
+symbol table version string. For example, for DF version 51.05, you would use
+these subdirectories:
+
+- "v0.51.05 linux64 CLASSIC"
+- "v0.51.05 linux64 ITCH"
+- "v0.51.05 linux64 STEAM"
+- "v0.51.05 win64 CLASSIC"
+- "v0.51.05 win64 ITCH"
+- "v0.51.05 win64 STEAM"
+
+See https://docs.dfhack.org/en/stable/docs/dev/Binpatches.html for more details.

docs/changelog.txt

@@ -57,15 +57,15 @@ Template for new versions:
 - `spectate`: can now specify number of seconds (in real time) before switching to follow a new unit
 - `spectate`: new "cinematic-action" mode that dynamically speeds up perspective switches based on intensity of conflict
 - `spectate`: new global keybinding for toggling spectate mode: Ctrl-Shift-S
-- `spectate`: new overlay panel that allows you to cycle through following next/prevous units (regardless of whether spectate mode is enabled)
-- ``disable-vanilla-dimensions``: new binpatch to disable the vanilla dimensions tooltip
+- `spectate`: new overlay panel that allows you to cycle through following next/previous units (regardless of whether spectate mode is enabled)
 - `gui/sitemap`: is now the official "go to" tool. new global hotkey for fort and adventure mode: Ctrl-G
 
 ## Fixes
 - Windows console: fix possible hang if the console returns a too-small window width (for any reason)
-- `createitem`: output items will now end up at look cursor if active
+- `createitem`: produced items will now end up at the look cursor position (if it is active)
 - `spectate`: don't allow temporarily modified announcement settings to be written to disk when "auto-unpause" mode is enabled
 - `changevein`: fix a crash that could occur when attempting to change a vein into itself
+- `overlay`: reset draw context between rendering widgets so context changes can't propagate from widget to widget
 
 ## Misc Improvements
 - `spectate`: player-set configuration is now stored globally instead of per-fort
@@ -77,22 +77,32 @@ Template for new versions:
 
 ## API
 - ``Military::removeFromSquad``: removes unit from any squad assignments
-- ``Buildings::checkFreeTiles``: now replaces the extents parameter for a building pointer
+- ``Buildings::checkFreeTiles``: now takes a building instead of a pointer to the building extents
 - ``Units::isUnitInBox``, ``Units::getUnitsInBox``: don't include inactive units
+- ``Items::getItemBaseValue``: adjust to the reduced value of prepared meals (changed in DF 51.06)
+- ``Items::getValue``: magical powers now correctly contribute to item value
 
 ## Lua
 - ``dfhack.units.setAutomaticProfessions``: sets unit labors according to current work detail settings
 - ``dfhack.military.removeFromSquad``: Lua API for ``Military::removeFromSquad``
 - ``gui.dwarfmode``: adventure mode cursor now supported in ``getCursorPos``, ``setCursorPos``, and ``clearCursorPos`` funcitons
-- ``dfhack.buildings.checkFreeTiles``: now replaces the extents parameter for a building pointer
+- ``dfhack.buildings.checkFreeTiles``: now takes a building pointer instead of an extents parameter
 - ``overlay.isOverlayEnabled``: new API for querying whether a given overlay is enabled
+- ``overlay``: widgets can now declare ``overlay_onenable`` and ``overlay_ondisable`` functions to hook enable/disable
 
 ## Removed
+- `orders`: MakeCheese job removed from library/basic orders set. Please use `autocheese` instead!
+
+# 51.06-r1
+
+## Misc Improvements
+- Compatibility with DF 51.06
 
 # 51.05-r1
 
 ## Misc Improvements
 - Compatibility with DF 51.05
+
 # 51.04-r1.1
 
 ## Fixes

docs/dev/Binpatches.rst

@@ -50,10 +50,20 @@ directly in memory at runtime::
 
 If the name of the patch has no extension or directory separators, the
 script uses :file:`hack/patches/<df-version>/<name>.dif`, thus auto-selecting
-the version appropriate for the currently loaded executable.
+the version appropriate for the currently loaded executable. The ``df-version``
+is the version string in the loaded symbol table. For example, if you want to
+make a patch for all distributed verisons of DF 51.05, you'd provide a ``dif``
+file in each of the following directories:
+
+- :file:`hack/patches/v0.51.05 linux64 CLASSIC/mypatch.dif`
+- :file:`hack/patches/v0.51.05 linux64 ITCH/mypatch.dif`
+- :file:`hack/patches/v0.51.05 linux64 STEAM/mypatch.dif`
+- :file:`hack/patches/v0.51.05 win64 CLASSIC/mypatch.dif`
+- :file:`hack/patches/v0.51.05 win64 ITCH/mypatch.dif`
+- :file:`hack/patches/v0.51.05 win64 STEAM/mypatch.dif`
 
 This is the preferred method; it's easier to debug, does not cause persistent
-problems, and leaves file checksums alone.  As with many other commands, users
+problems, and leaves file checksums alone. As with many other commands, users
 can simply add it to `dfhack.init` to reapply the patch every time DF is run.
 
 

docs/dev/overlay-dev-guide.rst

@@ -86,6 +86,11 @@ beyond your everyday `widgets.Widget <widget>`:
         end
 
     This allows for dynamic updates to UI overlays directly from the CLI.
+- If an ``overlay_onenable()`` function is defined, it is called when the
+    overlay is enabled (including when the persisted state is reloaded at DF
+    startup).
+- If an ``overlay_ondisable()`` function is defined, it is called when the
+    overlay is disabled.
 
 If the widget can take up a variable amount of space on the screen, and you want
 the widget to adjust its position according to the size of its contents, you can

docs/plugins/orders.rst

@@ -145,7 +145,14 @@ This collection of orders handles basic fort necessities:
 You should import it as soon as you have enough dwarves to perform the tasks.
 Right after the first migration wave is usually a good time.
 
-Note that the jugs are specifically made out of wood. This is so, as long as you don't may any other "Tools" out of wood, you can have a stockpile just for jugs by restricting a finished goods stockpile to only take wooden tools.
+These orders do not contain milking, shearing, or cheesemaking jobs since the
+game does not provide sufficient order conditions. Please enable ``automilk``,
+``autoshear``, and `autocheese` on the DFHack `gui/control-panel` for these
+types of jobs.
+
+Note that the jugs are specifically made out of wood. This is so, as long as
+you don't may any other "Tools" out of wood, you can have a stockpile just for
+jugs by restricting a finished goods stockpile to only take wooden tools.
 
 Armok's additional note: "shleggings? Yes,
 `shleggings <https://youtu.be/bLN8cOcTjdo&t=3458>`__."
@@ -155,7 +162,7 @@ Armok's additional note: "shleggings? Yes,
 
 This collection creates basic items that require heat. It is separated out from
 ``library/basic`` to give players the opportunity to set up magma furnaces first
-in order to save resources. It handles:
+(if desired) in order to save resources. It handles:
 
 - charcoal (including smelting of bituminous coal and lignite)
 - pearlash

library/Core.cpp

@@ -2987,7 +2987,7 @@ bool MemoryPatcher::verifyAccess(void *target, size_t count, bool write)
 
         save.push_back(perms);
         perms.write = perms.read = true;
-        if (!p->setPermisions(perms, perms))
+        if (!p->setPermissions(perms, perms))
             return false;
     }
 
@@ -3000,13 +3000,15 @@ bool MemoryPatcher::write(void *target, const void *src, size_t size)
         return false;
 
     memmove(target, src, size);
+
+    p->flushCache(target, size);
     return true;
 }
 
 void MemoryPatcher::close()
 {
     for (size_t i  = 0; i < save.size(); i++)
-        p->setPermisions(save[i], save[i]);
+        p->setPermissions(save[i], save[i]);
 
     save.clear();
     ranges.clear();

library/DataIdentity.cpp

@@ -38,6 +38,7 @@ namespace df {
 
     const bool_identity identity_traits<bool>::identity;
     const stl_string_identity identity_traits<std::string>::identity;
+    const path_identity identity_traits<std::filesystem::path>::identity;
     const ptr_string_identity identity_traits<char*>::identity;
     const ptr_string_identity identity_traits<const char*>::identity;
     const pointer_identity identity_traits<void*>::identity;

library/LuaTypes.cpp

@@ -188,6 +188,24 @@ void df::stl_string_identity::lua_write(lua_State *state, int fname_idx, void *p
     *(std::string*)ptr = std::string(bytes, size);
 }
 
+void df::path_identity::lua_read(lua_State* state, int fname_idx, void* ptr) const
+{
+    auto ppath = (std::filesystem::path*)ptr;
+    auto str = ppath->u8string();
+    lua_pushlstring(state, (char*)str.data(), str.size());
+}
+
+void df::path_identity::lua_write(lua_State* state, int fname_idx, void* ptr, int val_index) const
+{
+    size_t size;
+    const char* bytes = lua_tolstring(state, val_index, &size);
+    if (!bytes)
+        field_error(state, fname_idx, "path expected", "write");
+
+    std::u8string str((char8_t*)bytes, size);
+    *(std::filesystem::path*)ptr = std::filesystem::path(str);
+}
+
 void df::pointer_identity::lua_read(lua_State *state, int fname_idx, void *ptr, const type_identity *target)
 {
     push_object_internal(state, target, *(void**)ptr);

library/LuaWrapper.cpp

@@ -49,7 +49,7 @@ using namespace DFHack::LuaWrapper;
 /**
  * Report an error while accessing a field (index = field name).
  */
-void LuaWrapper::field_error(lua_State *state, int index, const char *err, const char *mode)
+[[noreturn]] void LuaWrapper::field_error(lua_State *state, int index, const char *err, const char *mode)
 {
     if (lua_islightuserdata(state, UPVAL_METATABLE))
         lua_pushstring(state, "(global)");
@@ -59,6 +59,7 @@ void LuaWrapper::field_error(lua_State *state, int index, const char *err, const
     const char *fname = index ? lua_tostring(state, index) : "*";
     luaL_error(state, "Cannot %s field %s.%s: %s.",
                mode, (cname ? cname : "?"), (fname ? fname : "?"), err);
+    std::abort(); // should never be reached but makes gcc happy
 }
 
 /* */

library/Process-linux.cpp

@@ -205,7 +205,7 @@ int Process::getPID()
     return getpid();
 }
 
-bool Process::setPermisions(const t_memrange & range,const t_memrange &trgrange)
+bool Process::setPermissions(const t_memrange & range,const t_memrange &trgrange)
 {
     int result;
     int protect=0;
@@ -217,6 +217,13 @@ bool Process::setPermisions(const t_memrange & range,const t_memrange &trgrange)
     return result==0;
 }
 
+bool Process::flushCache(const void* target, size_t count)
+{
+    __builtin___clear_cache((char*)target, (char*)target + count - 1);
+    return true; /* assume always succeeds, as the builtin has no return type */
+}
+
+
 // returns -1 on error
 void* Process::memAlloc(const int length)
 {

library/Process-windows.cpp

@@ -399,7 +399,7 @@ int Process::getPID()
 }
 
 
-bool Process::setPermisions(const t_memrange & range,const t_memrange &trgrange)
+bool Process::setPermissions(const t_memrange & range,const t_memrange &trgrange)
 {
     DWORD newprotect=0;
     if(trgrange.read && !trgrange.write && !trgrange.execute)newprotect=PAGE_READONLY;
@@ -414,6 +414,11 @@ bool Process::setPermisions(const t_memrange & range,const t_memrange &trgrange)
     return result;
 }
 
+bool Process::flushCache(const void* target, size_t count)
+{
+    return 0 != FlushInstructionCache(d->my_handle, (LPCVOID)target, count);
+}
+
 void* Process::memAlloc(const int length)
 {
     void *ret;

library/include/DataIdentity.h

@@ -34,6 +34,7 @@ distribution.
 #include <variant>
 #include <vector>
 #include <variant>
+#include <filesystem>
 
 #include "DataDefs.h"
 
@@ -298,6 +299,24 @@ namespace df
         virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const;
     };
 
+    class DFHACK_EXPORT path_identity : public DFHack::constructed_identity {
+    public:
+        path_identity()
+            : constructed_identity(sizeof(std::filesystem::path), &allocator_fn<std::filesystem::path>)
+        {
+        };
+
+        const std::string getFullName() const { return "path"; }
+
+        virtual DFHack::identity_type type() const { return DFHack::IDTYPE_PRIMITIVE; }
+
+        virtual bool isPrimitive() const { return true; }
+
+        virtual void lua_read(lua_State* state, int fname_idx, void* ptr) const;
+        virtual void lua_write(lua_State* state, int fname_idx, void* ptr, int val_index) const;
+    };
+
+
     class DFHACK_EXPORT stl_ptr_vector_identity : public ptr_container_identity {
     public:
         typedef std::vector<void*> container;
@@ -616,6 +635,11 @@ namespace df
         static const stl_string_identity *get() { return &identity; }
     };
 
+    template<> struct DFHACK_EXPORT identity_traits<std::filesystem::path> {
+        static const bool is_primitive = true;
+        static const path_identity identity;
+        static const path_identity* get() { return &identity; }
+    };
     template<> struct DFHACK_EXPORT identity_traits<char*> {
         static const bool is_primitive = true;
         static const ptr_string_identity identity;

library/include/LuaWrapper.h

@@ -142,7 +142,7 @@ namespace LuaWrapper {
     /**
     * Report an error while accessing a field (index = field name).
     */
-    void field_error(lua_State *state, int index, const char *err, const char *mode);
+    [[noreturn]] void field_error(lua_State *state, int index, const char *err, const char *mode);
 
     /*
      * If is_method is true, these use UPVAL_TYPETABLE to save a hash lookup.