Feature/197

.gitattributes

@@ -0,0 +1,4 @@
+# Set the default behavior, in case core.autocrlf isn't set.
+* text=auto
+
+test/tests/**/*.txt text eol=lf

.github/workflows/cmake-multi-platform.yml

@@ -43,7 +43,7 @@ jobs:
 
     - name: Install dependencies on Ubuntu
       if: matrix.os == 'ubuntu-latest'
-      run: sudo apt-get install -y libldap2-dev
+      run: sudo apt-get update && sudo apt-get install -y libldap2-dev
 
     - name: Configure CMake
       run: cmake -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} --preset=${{ matrix.cmake_preset }}

CHANGELOG.md

@@ -4,6 +4,7 @@
 #### Features
   - Contents of the cache file can now be printed in JSON format (#54)
   - New functionality for blacklisting users (#223)
+  - Space for the cache file is now pre-allocated before SCIM operations start (#197)
 
 ## v2.18 (2024-11-19)
 #### Features

CMakeLists.txt

@@ -35,7 +35,7 @@ else()
     include_directories(${CURL_INCLUDE_DIRS})
 endif ()
 
-find_package(Boost REQUIRED COMPONENTS program_options uuid)
+find_package(Boost REQUIRED COMPONENTS program_options uuid interprocess)
 if (NOT Boost_FOUND)
     message(FATAL_ERROR "please install boost")
 else ()
@@ -51,16 +51,16 @@ endif()
 link_libraries(${LDFLAGS})
 
 set(srcroot src)
-set(SOURCES ${srcroot}/scim_json_parse.cpp ${srcroot}/utility/simplescim_error_string.cpp ${srcroot}/simplescim_scim_send.cpp ${srcroot}/cache_file.cpp ${srcroot}/model/base_object.cpp ${srcroot}/model/object_list.cpp ${srcroot}/model/object_list.hpp ${srcroot}/simplescim_ldap_attrs_parser.cpp ${srcroot}/config_file.cpp ${srcroot}/config_file_parser.cpp ${srcroot}/scim.cpp ${srcroot}/utility/utils.cpp ${srcroot}/utility/utils.hpp ${srcroot}/utility/utils.cpp ${srcroot}/ldap_wrapper.cpp ${srcroot}/ldap_wrapper.hpp ${srcroot}/data_server.cpp ${srcroot}/data_server.hpp ${srcroot}/json_data_file.cpp ${srcroot}/json_data_file.hpp ${srcroot}/json_template_parser.cpp ${srcroot}/json_template_parser.hpp ${srcroot}/simplescim_ldap.cpp ${srcroot}/generated_load.cpp ${srcroot}/generated_load.hpp ${srcroot}/generated_group_load.cpp ${srcroot}/generated_group_load.hpp ${srcroot}/csv_load.cpp ${srcroot}/csv_load.hpp ${srcroot}/csv_store.cpp ${srcroot}/csv_store.hpp ${srcroot}/csv_file.cpp ${srcroot}/csv_file.hpp ${srcroot}/sql_load.cpp ${srcroot}/sql_load.hpp ${srcroot}/scim_server_info.cpp ${srcroot}/scim_server_info.hpp ${srcroot}/load_limiter.hpp ${srcroot}/load_limiter.cpp ${srcroot}/fedtlsauth/castore_file.hpp ${srcroot}/fedtlsauth/castore_file.cpp ${srcroot}/fedtlsauth/metadata_parser.hpp ${srcroot}/fedtlsauth/metadata_parser.cpp ${srcroot}/utility/indented_logger.hpp ${srcroot}/utility/indented_logger.cpp ${srcroot}/post_processing.hpp ${srcroot}/post_processing.cpp ${srcroot}/sql.hpp ${srcroot}/sql.cpp ${srcroot}/dl.hpp ${srcroot}/dl.cpp ${srcroot}/plugin_config.hpp ${srcroot}/plugin_config.cpp ${srcroot}/load_common.hpp ${srcroot}/load_common.cpp ${srcroot}/load_limiter_impl.hpp ${srcroot}/model/rendered_object.hpp ${srcroot}/model/rendered_object.cpp ${srcroot}/model/rendered_object_list.hpp ${srcroot}/model/rendered_object_list.cpp ${srcroot}/renderer.hpp ${srcroot}/renderer.cpp ${srcroot}/rendered_cache_file.hpp ${srcroot}/rendered_cache_file.cpp ${srcroot}/utility/temporary_umask.hpp ${srcroot}/transformer.hpp ${srcroot}/transformer.cpp ${srcroot}/transformer_impl.hpp ${srcroot}/transformer_impl.cpp ${srcroot}/readable_id.hpp ${srcroot}/readable_id.cpp ${srcroot}/config.hpp ${srcroot}/config.cpp ${srcroot}/dnpactivities.cpp  ${srcroot}/dnpactivities.hpp ${srcroot}/utility/simpleurl.cpp ${srcroot}/utility/simpleurl.hpp ${srcroot}/thresholds.hpp ${srcroot}/thresholds.cpp ${srcroot}/audit.hpp ${srcroot}/audit.cpp ${srcroot}/print_cache.hpp ${srcroot}/print_cache.cpp)
+set(SOURCES ${srcroot}/scim_json_parse.cpp ${srcroot}/utility/simplescim_error_string.cpp ${srcroot}/simplescim_scim_send.cpp ${srcroot}/cache_file.cpp ${srcroot}/model/base_object.cpp ${srcroot}/model/object_list.cpp ${srcroot}/model/object_list.hpp ${srcroot}/simplescim_ldap_attrs_parser.cpp ${srcroot}/config_file.cpp ${srcroot}/config_file_parser.cpp ${srcroot}/scim.cpp ${srcroot}/utility/utils.cpp ${srcroot}/utility/utils.hpp ${srcroot}/utility/utils.cpp ${srcroot}/ldap_wrapper.cpp ${srcroot}/ldap_wrapper.hpp ${srcroot}/data_server.cpp ${srcroot}/data_server.hpp ${srcroot}/json_data_file.cpp ${srcroot}/json_data_file.hpp ${srcroot}/json_template_parser.cpp ${srcroot}/json_template_parser.hpp ${srcroot}/simplescim_ldap.cpp ${srcroot}/generated_load.cpp ${srcroot}/generated_load.hpp ${srcroot}/generated_group_load.cpp ${srcroot}/generated_group_load.hpp ${srcroot}/csv_load.cpp ${srcroot}/csv_load.hpp ${srcroot}/csv_store.cpp ${srcroot}/csv_store.hpp ${srcroot}/csv_file.cpp ${srcroot}/csv_file.hpp ${srcroot}/sql_load.cpp ${srcroot}/sql_load.hpp ${srcroot}/scim_server_info.cpp ${srcroot}/scim_server_info.hpp ${srcroot}/load_limiter.hpp ${srcroot}/load_limiter.cpp ${srcroot}/fedtlsauth/castore_file.hpp ${srcroot}/fedtlsauth/castore_file.cpp ${srcroot}/fedtlsauth/metadata_parser.hpp ${srcroot}/fedtlsauth/metadata_parser.cpp ${srcroot}/utility/indented_logger.hpp ${srcroot}/utility/indented_logger.cpp ${srcroot}/post_processing.hpp ${srcroot}/post_processing.cpp ${srcroot}/sql.hpp ${srcroot}/sql.cpp ${srcroot}/dl.hpp ${srcroot}/dl.cpp ${srcroot}/plugin_config.hpp ${srcroot}/plugin_config.cpp ${srcroot}/load_common.hpp ${srcroot}/load_common.cpp ${srcroot}/load_limiter_impl.hpp ${srcroot}/model/rendered_object.hpp ${srcroot}/model/rendered_object.cpp ${srcroot}/model/rendered_object_list.hpp ${srcroot}/model/rendered_object_list.cpp ${srcroot}/renderer.hpp ${srcroot}/renderer.cpp ${srcroot}/rendered_cache_file.hpp ${srcroot}/rendered_cache_file.cpp ${srcroot}/utility/temporary_umask.hpp ${srcroot}/transformer.hpp ${srcroot}/transformer.cpp ${srcroot}/transformer_impl.hpp ${srcroot}/transformer_impl.cpp ${srcroot}/readable_id.hpp ${srcroot}/readable_id.cpp ${srcroot}/config.hpp ${srcroot}/config.cpp ${srcroot}/dnpactivities.cpp  ${srcroot}/dnpactivities.hpp ${srcroot}/utility/simpleurl.cpp ${srcroot}/utility/simpleurl.hpp ${srcroot}/thresholds.hpp ${srcroot}/thresholds.cpp ${srcroot}/audit.hpp ${srcroot}/audit.cpp ${srcroot}/print_cache.hpp ${srcroot}/print_cache.cpp ${srcroot}/advisory_file_lock.cpp ${srcroot}/advisory_file_lock.hpp)
 
-set(TEST_SOURCES ${srcroot}/tests/main.cpp ${srcroot}/tests/json_template_parser_tests.cpp ${srcroot}/tests/scim_query_tests.cpp ${srcroot}/tests/csv_tests.cpp ${srcroot}/tests/uuid_tests.cpp ${srcroot}/tests/scim_json_parse_tests.cpp ${srcroot}/tests/load_limiter_tests.cpp ${srcroot}/tests/object_list_tests.cpp ${srcroot}/tests/generated_group_tests.cpp ${srcroot}/tests/transformer_tests.cpp ${srcroot}/tests/dnpactivities_tests.cpp ${srcroot}/tests/thresholds_tests.cpp ${srcroot}/tests/audit_tests.cpp)
+set(TEST_SOURCES ${srcroot}/tests/main.cpp ${srcroot}/tests/json_template_parser_tests.cpp ${srcroot}/tests/scim_query_tests.cpp ${srcroot}/tests/csv_tests.cpp ${srcroot}/tests/uuid_tests.cpp ${srcroot}/tests/scim_json_parse_tests.cpp ${srcroot}/tests/load_limiter_tests.cpp ${srcroot}/tests/object_list_tests.cpp ${srcroot}/tests/generated_group_tests.cpp ${srcroot}/tests/transformer_tests.cpp ${srcroot}/tests/dnpactivities_tests.cpp ${srcroot}/tests/thresholds_tests.cpp ${srcroot}/tests/audit_tests.cpp ${srcroot}/tests/rendered_cache_file_tests.cpp)
 
 add_library (Egil ${SOURCES})
 
 add_executable(EgilSCIMClient ${srcroot}/main.cpp)
 add_executable(tests ${TEST_SOURCES})
 
-target_link_libraries(EgilSCIMClient LINK_PUBLIC Egil Boost::program_options Boost::uuid CURL::libcurl)
+target_link_libraries(EgilSCIMClient LINK_PUBLIC Egil Boost::program_options Boost::uuid CURL::libcurl Boost::interprocess)
 target_link_libraries(tests LINK_PUBLIC Egil)
 
 install (TARGETS EgilSCIMClient DESTINATION bin)

src/advisory_file_lock.cpp

@@ -0,0 +1,69 @@
+/**
+ *  This file is part of the EGIL SCIM client.
+ *
+ *  Copyright (C) 2017-2025 Föreningen Sambruk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "advisory_file_lock.hpp"
+#include "utility/utils.hpp"
+#include <filesystem>
+#include <boost/date_time/posix_time/posix_time.hpp>
+
+namespace {
+    /** Construct a name for the mutex based on the path.
+    *   We'll canonicalize the path in case it has been given differently
+    *   to different processes. We'll then generate a UUID based on that
+    *   since path names are not valid mutex names (at least on Windows).
+    */
+    std::string mutex_name(const std::string& path) {
+        std::string str;
+        try {
+            str = std::filesystem::weakly_canonical(std::filesystem::path(path)).string();
+        }
+        catch (const std::exception&) {
+            str = path;
+        }
+        return uuid_util::instance().generate(str);
+    }
+}
+
+AdvisoryFileLock::AdvisoryFileLock(const std::string& path, int timeout)
+ : m_timeout(timeout) {
+    // Creating or locking the mutex might throw in exotic situations,
+    // we don't want that to block us from writing the cache file.
+    try {
+        const auto mtx_name = mutex_name(path);
+        m_pmtx = std::make_unique<boost::interprocess::named_mutex>(boost::interprocess::open_or_create_t{}, mtx_name.c_str());
+        auto locked = m_pmtx->timed_lock(boost::posix_time::second_clock::universal_time() + boost::posix_time::seconds(m_timeout));
+        if (!locked) {
+            // There really shouldn't be any congestion for this mutex, if we fail to lock
+            // the most likely explanation is a problem with the mutex (perhaps a process that held
+            // the mutex was hard killed before it could release the mutex).
+            m_pmtx = nullptr;
+            boost::interprocess::named_mutex::remove(mtx_name.c_str());
+        }
+    }
+    catch (std::exception&) {
+        m_pmtx = nullptr;
+    }
+
+}
+
+AdvisoryFileLock::~AdvisoryFileLock() {
+    if (m_pmtx != nullptr) {
+        m_pmtx->unlock();
+    }
+}

src/advisory_file_lock.hpp

@@ -0,0 +1,71 @@
+/**
+ *  This file is part of the EGIL SCIM client.
+ *
+ *  Copyright (C) 2017-2025 Föreningen Sambruk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef EGILSCIM_ADVISORY_FILE_LOCK_HPP
+#define EGILSCIM_ADVISORY_FILE_LOCK_HPP
+
+#include <memory>
+#include <string>
+#include <boost/interprocess/sync/named_mutex.hpp>
+
+/** This is an advisory file lock for the cache file.
+  * Internally this uses boost's named_mutex to coordinate
+  * access to the cache file, but in a pragmatic way to
+  * suit our needs for the cache file.
+  * 
+  * Motivation:
+  * 
+  * 1. We really want to avoid being blocked from writing the new cache file.
+  * 2. Under normal circumstances there should be little risk of having to 
+  *    wait long for access to the file.
+  * 3. There could be other processes accessing the file (such as a backup script)
+  *    which won't respect our advisory locking anyway, so the writing to the
+  *    cache file will anyway need to implement a retry in case the filesystem
+  *    blocks the rename when we're done writing.
+  * 4. boost's named_mutex can throw an interprocess_error, if the interprocess
+  *    implementation isn't working for some reason we don't want that to be
+  *    enough of a reason not to write the cache file.
+  * 5. If we fail to get the lock within a reasonable time, the most likely explanation
+  *    is not that others are accessing it, but that the named_mutex hasn't been properly
+  *    released (for instance it the process is hard killed while holding the mutex).
+  * 
+  * All this put together leads to an implementation which tries to take the
+  * named mutex but with a timeout. If we fail to take the mutex (either by timeout or
+  * a problem with the interprocess system), we'll carry on as if we have the mutex
+  * (and remove it from the system so we won't need to wait next time).
+  * 
+  * In other words, under normal circumstances this will block multiple instances of
+  * EgilSCIMClient from accessing the file at once, but it's not a 100% guarantee.
+  * We're ok with that since we're anyway retrying the writes.
+  */
+class AdvisoryFileLock {
+public:
+    /*
+     * path is the file to lock.
+     * timeout is given in seconds.
+     */
+    AdvisoryFileLock(const std::string& path, int timeout);
+    virtual ~AdvisoryFileLock();
+
+private:
+    std::unique_ptr<boost::interprocess::named_mutex> m_pmtx;
+    int m_timeout;
+};
+
+#endif // EGILSCIM_ADVISORY_FILE_LOCK_HPP