initial implementation of Files subsystem

ci/run-tests.py

@@ -72,6 +72,11 @@ def change_setting(content, setting, value):
 #if args.headless:
 #    init_contents = change_setting(init_contents, 'PRINT_MODE', 'TEXT')
 
+# force portable mode for testing so that anything in the user prefs doesn't interfere
+portable_txt_path = 'prefs/portable.txt'
+with open(portable_txt_path, 'w') as f:
+    pass  # empty file to enable portable mode
+
 init_path = 'dfhack-config/init'
 if not os.path.isdir('hack/init'):
     # we're on an old branch that still reads init files from the root dir

library/CMakeLists.txt

@@ -59,6 +59,7 @@ set(MAIN_HEADERS
     include/DebugManager.h
     include/Error.h
     include/Export.h
+    include/Files.h
     include/Format.h
     include/Hooks.h
     include/LuaTools.h
@@ -89,33 +90,34 @@ set(MAIN_HEADERS_WINDOWS
 )
 
 set(MAIN_SOURCES
-    Core.cpp
     ColorText.cpp
     Commands.cpp
     CompilerWorkAround.cpp
+    Core.cpp
     DataDefs.cpp
     DataIdentity.cpp
+    DataStatics.cpp
+    DataStaticsCtor.cpp
     Debug.cpp
     Error.cpp
-    VTableInterpose.cpp
-    LuaWrapper.cpp
-    LuaTypes.cpp
-    LuaTools.cpp
+    Files.cpp
     LuaApi.cpp
-    DataStatics.cpp
-    DataStaticsCtor.cpp
+    LuaTools.cpp
+    LuaTypes.cpp
+    LuaWrapper.cpp
     MemoryPatcher.cpp
     MiscUtils.cpp
-    Types.cpp
     PluginManager.cpp
     PluginStatics.cpp
     PlugLoad.cpp
     Process.cpp
-    TileTypes.cpp
-    VersionInfoFactory.cpp
     RemoteClient.cpp
     RemoteServer.cpp
     RemoteTools.cpp
+    TileTypes.cpp
+    Types.cpp
+    VersionInfoFactory.cpp
+    VTableInterpose.cpp
 )
 
 file(GLOB_RECURSE TEST_SOURCES

library/Commands.cpp

@@ -434,7 +434,7 @@ namespace DFHack
     {
         if (parts.size() == 1)
         {
-            core.loadScriptFile(con, std::filesystem::weakly_canonical(std::filesystem::path{parts[0]}), false);
+            core.loadScriptFile(con, std::filesystem::path{parts[0]}.lexically_normal(), false);
             return CR_OK;
         }
         else

library/Core.cpp

@@ -27,6 +27,7 @@ distribution.
 #include "Internal.h"
 
 #include "Error.h"
+#include "Files.h"
 #include "MemAccess.h"
 #include "DataDefs.h"
 #include "Debug.h"
@@ -117,16 +118,6 @@ namespace DFHack {
     DBG_DECLARE(core, keybinding, DebugCategory::LINFO);
     DBG_DECLARE(core, script, DebugCategory::LINFO);
 
-    static const std::filesystem::path getConfigPath()
-    {
-        return Filesystem::getInstallDir() / "dfhack-config";
-    };
-
-    static const std::filesystem::path getConfigDefaultsPath()
-    {
-        return Filesystem::getInstallDir() / "hack" / "data" / "dfhack-config-defaults";
-    };
-
 class MainThread {
 public:
     //! MainThread::suspend keeps the main DF thread suspended from Core::Init to
@@ -442,8 +433,10 @@ static bool try_autocomplete(color_ostream &con, const std::string &first, std::
 
 bool Core::addScriptPath(std::filesystem::path path, bool search_before)
 {
-    std::lock_guard<std::mutex> lock(script_path_mutex);
-    auto &vec = script_paths[search_before ? 0 : 1];
+    std::lock_guard lock(script_path_mutex);
+
+    auto& vec = search_before ? script_paths_first : script_paths_last;
+
     if (std::find(vec.begin(), vec.end(), path) != vec.end())
         return false;
     if (!Filesystem::isdir(path))
@@ -452,57 +445,70 @@ bool Core::addScriptPath(std::filesystem::path path, bool search_before)
     return true;
 }
 
-bool Core::setModScriptPaths(const std::vector<std::filesystem::path> &mod_script_paths) {
-    std::lock_guard<std::mutex> lock(script_path_mutex);
-    script_paths[2] = mod_script_paths;
+bool Core::setModScriptPaths(const Core::filelist &mod_script_paths) {
+    std::lock_guard lock(script_path_mutex);
+
+    script_paths_mod = mod_script_paths;
     return true;
 }
 
-bool Core::removeScriptPath(std::filesystem::path path)
+bool Core::removeScriptPath(std::filesystem::path& path)
 {
-    std::lock_guard<std::mutex> lock(script_path_mutex);
+    auto pathlists = {&script_paths_first, &script_paths_last, &script_paths_mod};
+
+    std::lock_guard lock(script_path_mutex);
+
     bool found = false;
-    for (int i = 0; i < 2; i++)
+
+    for (auto& vec : pathlists)
     {
-        auto &vec = script_paths[i];
-        while (1)
-        {
-            auto it = std::find(vec.begin(), vec.end(), path);
-            if (it == vec.end())
-                break;
-            vec.erase(it);
+        auto cnt = std::erase(*vec, path);
+        if (cnt > 0)
             found = true;
-        }
     }
+
     return found;
 }
 
-void Core::getScriptPaths(std::vector<std::filesystem::path> *dest)
+Core::filelist Core::getScriptPaths()
 {
-    std::lock_guard<std::mutex> lock(script_path_mutex);
-    dest->clear();
-    std::filesystem::path df_pref_path = Filesystem::getBaseDir();
-    std::filesystem::path df_install_path = Filesystem::getInstallDir();
-    for (auto & path : script_paths[0])
-        dest->emplace_back(path);
-    // should this be df_pref_path? probably
-    dest->push_back(getConfigPath() / "scripts");
-    if (df::global::world && isWorldLoaded()) {
+    using path = std::filesystem::path;
+
+    std::lock_guard lock(script_path_mutex);
+
+    filelist dest;
+    dest.clear();
+
+    for (auto & path : script_paths_first)
+        dest.emplace_back(path);
+
+    filelist candidates{path{"dfhack-config"} / "scripts"};
+
+    if (df::global::world && isWorldLoaded())
+    {
         std::string save = World::ReadWorldFolder();
         if (save.size())
-            dest->emplace_back(df_pref_path / "save" / save / "scripts");
+            candidates.push_back(path{"save"} / save / "scripts");
     }
-    dest->emplace_back(df_install_path / "hack" / "scripts");
-    for (auto & path : script_paths[2])
-        dest->emplace_back(path);
-    for (auto & path : script_paths[1])
-        dest->emplace_back(path);
+
+    for (auto& cand : candidates)
+        for (auto& path : getPossiblePaths(cand))
+            dest.push_back(path);
+
+    dest.emplace_back(getInstallPath() / "hack" / "scripts");
+
+    for (auto & path : script_paths_mod)
+        dest.push_back(path);
+
+    for (auto & path : script_paths_last)
+        dest.push_back(path);
+
+    return dest;
 }
 
 std::filesystem::path Core::findScript(std::string name)
 {
-    std::vector<std::filesystem::path> paths;
-    getScriptPaths(&paths);
+    auto paths = getScriptPaths();
     for (auto& path : paths)
     {
         std::error_code ec;
@@ -522,8 +528,9 @@ std::filesystem::path Core::findScript(std::string name)
 
 bool loadScriptPaths(color_ostream &out, bool silent = false)
 {
-    std::filesystem::path filename{ getConfigPath() / "script-paths.txt" };
-    std::ifstream file(filename);
+    constexpr auto filename = "script-paths.txt";
+    auto file = openConfigFile(filename);
+
     if (!file)
     {
         if (!silent)
@@ -859,13 +866,15 @@ bool Core::loadScriptFile(color_ostream &out, std::filesystem::path fname, bool
         INFO(script,out) << "Running script: " << fname << std::endl;
         std::cerr << "Running script: " << fname << std::endl;
     }
-    std::ifstream script{ fname.c_str() };
+
+    auto script = openFile(fname);
     if ( !script )
     {
         if(!silent)
             out.printerr("Error loading script: {}\n", fname);
         return false;
     }
+
     std::string command;
     while(script.good()) {
         std::string temp;

library/Files.cpp

@@ -0,0 +1,93 @@
+#include <filesystem>
+#include <fstream>
+#include <vector>
+
+#include "modules/DFSDL.h"
+#include "Files.h"
+
+#include "df/init.h"
+#include "df/init_media_flags.h"
+
+namespace DFHack
+{
+    namespace fs = std::filesystem;
+
+    const fs::path getInstallPath() noexcept
+    {
+        return DFSDL::DFSDL_GetBasePath();
+    };
+
+    const fs::path getUserPath() noexcept
+    {
+        return DFSDL::DFSDL_GetPrefPath("Bay 12 Games", "Dwarf Fortress");
+    };
+
+    const bool isPortableMode() noexcept
+    {
+        return !df::global::init || df::global::init->media.flag.is_set(df::enums::init_media_flags::PORTABLE_MODE);
+    }
+
+    const fs::path getBasePath(bool portable) noexcept
+    {
+        return portable ? getInstallPath() : getUserPath();
+    }
+
+    const fs::path getBasePath() noexcept
+    {
+        return getBasePath(isPortableMode());
+    }
+
+    const fs::path getAltBasePath() noexcept
+    {
+        return getBasePath(!isPortableMode());
+    }
+
+    const fs::path getConfigPath(bool portable) noexcept
+    {
+        return getBasePath(portable) / "dfhack-config";
+    };
+
+    const fs::path getConfigDefaultsPath() noexcept
+    {
+        return getInstallPath() / "hack" / "data" / "dfhack-config-defaults";
+    }
+
+    const std::vector<fs::path> getPossiblePaths(const fs::path filename) noexcept
+    {
+        bool portable = isPortableMode();
+        return {getBasePath(portable) / filename, getBasePath(!portable) / filename};
+    }
+
+    std::ifstream openConfigFile(fs::path filename)
+    {
+        return openFile("dfhack-config" / filename);
+    }
+
+    const fs::path findConfigFile(fs::path filename)
+    {
+        return findFile("dfhack-config" / filename);
+    }
+
+    std::ifstream openFile(const fs::path filename)
+    {
+        auto path = findFile(filename);
+        if (path.empty())
+            return std::ifstream();
+        return std::ifstream(path);
+    }
+
+    const fs::path findFile(const fs::path filename)
+    {
+        if (filename.is_absolute())
+            return filename;
+
+        fmt::print(stderr, "Trying to locate file: {}\n", filename.string());
+
+        for (const auto& path : getPossiblePaths(filename))
+        {
+            if (fs::exists(path))
+                return path;
+        }
+        return fs::path();
+    }
+}

library/LuaApi.cpp

@@ -4099,7 +4099,8 @@ static int internal_addScriptPath(lua_State *L)
 
 static int internal_removeScriptPath(lua_State *L)
 {
-    const char *path = luaL_checkstring(L, 1);
+    const char *pathStr = luaL_checkstring(L, 1);
+    std::filesystem::path path = pathStr;
     lua_pushboolean(L, Core::getInstance().removeScriptPath(path));
     return 1;
 }
@@ -4108,8 +4109,7 @@ static int internal_getScriptPaths(lua_State *L)
 {
     int i = 1;
     lua_newtable(L);
-    vector<std::filesystem::path> paths;
-    Core::getInstance().getScriptPaths(&paths);
+    auto paths = Core::getInstance().getScriptPaths();
     for (auto it = paths.begin(); it != paths.end(); ++it)
     {
         lua_pushinteger(L, i++);

library/include/Core.h

@@ -176,9 +176,12 @@ namespace DFHack
 
         bool addScriptPath(std::filesystem::path path, bool search_before = false);
         bool setModScriptPaths(const std::vector<std::filesystem::path> & mod_script_paths);
-        bool removeScriptPath(std::filesystem::path path);
+        bool removeScriptPath(std::filesystem::path& path);
         std::filesystem::path findScript(std::string name);
-        void getScriptPaths(std::vector<std::filesystem::path> *dest);
+
+        using filelist = std::vector<std::filesystem::path>;
+
+        filelist getScriptPaths();
 
         bool getSuppressDuplicateKeyboardEvents() const;
         void setSuppressDuplicateKeyboardEvents(bool suppress);
@@ -308,7 +311,10 @@ namespace DFHack
         // Hotkey Manager
         DFHack::HotkeyManager *hotkey_mgr;
 
-        std::vector<std::filesystem::path> script_paths[3];
+        filelist script_paths_first;
+        filelist script_paths_mod;
+        filelist script_paths_last;
+
         std::mutex script_path_mutex;
 
         int8_t modstate;