Merge pull request #225 from Sambruk/feature/54

joesiltberg · web-flow · commit 6011da293474 · 2025-01-22T11:08:40.000+01:00
Adds functionality for printing the cache file.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Unreleased
 #### Features
+  - Contents of the cache file can now be printed in JSON format (#54)
   - New functionality for blacklisting users (#223)
 
 ## v2.18 (2024-11-19)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -51,7 +51,7 @@ endif()
 link_libraries(${LDFLAGS})
 
 set(srcroot src)
-set(SOURCES ${srcroot}/scim_json_parse.cpp ${srcroot}/utility/simplescim_error_string.cpp ${srcroot}/simplescim_scim_send.cpp ${srcroot}/cache_file.cpp ${srcroot}/model/base_object.cpp ${srcroot}/model/object_list.cpp ${srcroot}/model/object_list.hpp ${srcroot}/simplescim_ldap_attrs_parser.cpp ${srcroot}/config_file.cpp ${srcroot}/config_file_parser.cpp ${srcroot}/scim.cpp ${srcroot}/utility/utils.cpp ${srcroot}/utility/utils.hpp ${srcroot}/utility/utils.cpp ${srcroot}/ldap_wrapper.cpp ${srcroot}/ldap_wrapper.hpp ${srcroot}/data_server.cpp ${srcroot}/data_server.hpp ${srcroot}/json_data_file.cpp ${srcroot}/json_data_file.hpp ${srcroot}/json_template_parser.cpp ${srcroot}/json_template_parser.hpp ${srcroot}/simplescim_ldap.cpp ${srcroot}/generated_load.cpp ${srcroot}/generated_load.hpp ${srcroot}/generated_group_load.cpp ${srcroot}/generated_group_load.hpp ${srcroot}/csv_load.cpp ${srcroot}/csv_load.hpp ${srcroot}/csv_store.cpp ${srcroot}/csv_store.hpp ${srcroot}/csv_file.cpp ${srcroot}/csv_file.hpp ${srcroot}/sql_load.cpp ${srcroot}/sql_load.hpp ${srcroot}/scim_server_info.cpp ${srcroot}/scim_server_info.hpp ${srcroot}/load_limiter.hpp ${srcroot}/load_limiter.cpp ${srcroot}/fedtlsauth/castore_file.hpp ${srcroot}/fedtlsauth/castore_file.cpp ${srcroot}/fedtlsauth/metadata_parser.hpp ${srcroot}/fedtlsauth/metadata_parser.cpp ${srcroot}/utility/indented_logger.hpp ${srcroot}/utility/indented_logger.cpp ${srcroot}/post_processing.hpp ${srcroot}/post_processing.cpp ${srcroot}/sql.hpp ${srcroot}/sql.cpp ${srcroot}/dl.hpp ${srcroot}/dl.cpp ${srcroot}/plugin_config.hpp ${srcroot}/plugin_config.cpp ${srcroot}/load_common.hpp ${srcroot}/load_common.cpp ${srcroot}/load_limiter_impl.hpp ${srcroot}/model/rendered_object.hpp ${srcroot}/model/rendered_object.cpp ${srcroot}/model/rendered_object_list.hpp ${srcroot}/model/rendered_object_list.cpp ${srcroot}/renderer.hpp ${srcroot}/renderer.cpp ${srcroot}/rendered_cache_file.hpp ${srcroot}/rendered_cache_file.cpp ${srcroot}/utility/temporary_umask.hpp ${srcroot}/transformer.hpp ${srcroot}/transformer.cpp ${srcroot}/transformer_impl.hpp ${srcroot}/transformer_impl.cpp ${srcroot}/readable_id.hpp ${srcroot}/readable_id.cpp ${srcroot}/config.hpp ${srcroot}/config.cpp ${srcroot}/dnpactivities.cpp  ${srcroot}/dnpactivities.hpp ${srcroot}/utility/simpleurl.cpp ${srcroot}/utility/simpleurl.hpp ${srcroot}/thresholds.hpp ${srcroot}/thresholds.cpp ${srcroot}/audit.hpp ${srcroot}/audit.cpp)
+set(SOURCES ${srcroot}/scim_json_parse.cpp ${srcroot}/utility/simplescim_error_string.cpp ${srcroot}/simplescim_scim_send.cpp ${srcroot}/cache_file.cpp ${srcroot}/model/base_object.cpp ${srcroot}/model/object_list.cpp ${srcroot}/model/object_list.hpp ${srcroot}/simplescim_ldap_attrs_parser.cpp ${srcroot}/config_file.cpp ${srcroot}/config_file_parser.cpp ${srcroot}/scim.cpp ${srcroot}/utility/utils.cpp ${srcroot}/utility/utils.hpp ${srcroot}/utility/utils.cpp ${srcroot}/ldap_wrapper.cpp ${srcroot}/ldap_wrapper.hpp ${srcroot}/data_server.cpp ${srcroot}/data_server.hpp ${srcroot}/json_data_file.cpp ${srcroot}/json_data_file.hpp ${srcroot}/json_template_parser.cpp ${srcroot}/json_template_parser.hpp ${srcroot}/simplescim_ldap.cpp ${srcroot}/generated_load.cpp ${srcroot}/generated_load.hpp ${srcroot}/generated_group_load.cpp ${srcroot}/generated_group_load.hpp ${srcroot}/csv_load.cpp ${srcroot}/csv_load.hpp ${srcroot}/csv_store.cpp ${srcroot}/csv_store.hpp ${srcroot}/csv_file.cpp ${srcroot}/csv_file.hpp ${srcroot}/sql_load.cpp ${srcroot}/sql_load.hpp ${srcroot}/scim_server_info.cpp ${srcroot}/scim_server_info.hpp ${srcroot}/load_limiter.hpp ${srcroot}/load_limiter.cpp ${srcroot}/fedtlsauth/castore_file.hpp ${srcroot}/fedtlsauth/castore_file.cpp ${srcroot}/fedtlsauth/metadata_parser.hpp ${srcroot}/fedtlsauth/metadata_parser.cpp ${srcroot}/utility/indented_logger.hpp ${srcroot}/utility/indented_logger.cpp ${srcroot}/post_processing.hpp ${srcroot}/post_processing.cpp ${srcroot}/sql.hpp ${srcroot}/sql.cpp ${srcroot}/dl.hpp ${srcroot}/dl.cpp ${srcroot}/plugin_config.hpp ${srcroot}/plugin_config.cpp ${srcroot}/load_common.hpp ${srcroot}/load_common.cpp ${srcroot}/load_limiter_impl.hpp ${srcroot}/model/rendered_object.hpp ${srcroot}/model/rendered_object.cpp ${srcroot}/model/rendered_object_list.hpp ${srcroot}/model/rendered_object_list.cpp ${srcroot}/renderer.hpp ${srcroot}/renderer.cpp ${srcroot}/rendered_cache_file.hpp ${srcroot}/rendered_cache_file.cpp ${srcroot}/utility/temporary_umask.hpp ${srcroot}/transformer.hpp ${srcroot}/transformer.cpp ${srcroot}/transformer_impl.hpp ${srcroot}/transformer_impl.cpp ${srcroot}/readable_id.hpp ${srcroot}/readable_id.cpp ${srcroot}/config.hpp ${srcroot}/config.cpp ${srcroot}/dnpactivities.cpp  ${srcroot}/dnpactivities.hpp ${srcroot}/utility/simpleurl.cpp ${srcroot}/utility/simpleurl.hpp ${srcroot}/thresholds.hpp ${srcroot}/thresholds.cpp ${srcroot}/audit.hpp ${srcroot}/audit.cpp ${srcroot}/print_cache.hpp ${srcroot}/print_cache.cpp)
 
 set(TEST_SOURCES ${srcroot}/tests/main.cpp ${srcroot}/tests/json_template_parser_tests.cpp ${srcroot}/tests/scim_query_tests.cpp ${srcroot}/tests/csv_tests.cpp ${srcroot}/tests/uuid_tests.cpp ${srcroot}/tests/scim_json_parse_tests.cpp ${srcroot}/tests/load_limiter_tests.cpp ${srcroot}/tests/object_list_tests.cpp ${srcroot}/tests/generated_group_tests.cpp ${srcroot}/tests/transformer_tests.cpp ${srcroot}/tests/dnpactivities_tests.cpp ${srcroot}/tests/thresholds_tests.cpp ${srcroot}/tests/audit_tests.cpp)
 
diff --git a/doc/USAGE.md b/doc/USAGE.md
@@ -1124,3 +1124,43 @@ we are sending the operation to. Instead you can use a file path that shows what
 the destination is. Note that if you wish to re-use the same configuration files
 for multiple destinations you can use the -D command line argument to set the
 `audit-log-file` variable.
+
+## Inspecing the cache file
+Contents of the cache file can be printed to standard output by running the
+EgilSCIMClient with the `--print-cache` argument. For instance:
+
+```
+EgilSCIMClient --print-cache master.conf
+```
+
+The cache file configured in the specified configuration file will be printed
+in JSON format to standard output. As usual you can override the path to the
+cache file with the `--cache-file` argument. You still need to specify the
+configuration file.
+
+If you don't want to print all objects in the cache file you can limit them
+by type and/or attribute values. For instance, to print all `Student` objects:
+
+```
+EgilSCIMClient --print-cache --print-cache-type Student master.conf
+```
+
+Or to print a specific student:
+
+```
+EgilSCIMClient --print-cache --print-cache-type Student --print-cache-where userName=babs@example.com master.conf
+```
+
+If you wish to print all types of users it can be helpful to specify SCIM
+endpoints rather than EGIL types:
+
+```
+EgilSCIMClient --print-cache --print-cache-by-endpoint --print-cache-type Users master.conf
+```
+
+If you wish to search for a sub-attribute you can specify a path in the JSON
+object by using '/' as the separator:
+
+```
+EgilSCIMClient --print-cache  --print-cache-where 'name/familyName=Johansson'  master.conf
+```
diff --git a/src/main.cpp b/src/main.cpp
@@ -41,10 +41,24 @@
 #include "sql.hpp"
 #include "generated_group_load.hpp"
 #include "thresholds.hpp"
+#include "print_cache.hpp"
 
 namespace po = boost::program_options;
 namespace filesystem = std::experimental::filesystem;
 
+// TODO: Add the rest of the options names here
+namespace options {
+    const char* CACHE_FILE = "cache-file";
+
+    const char* USER_BLACKLIST_FILE = "user-blacklist-file";
+    const char* USER_BLACKLIST_ATTRIBUTE = "user-blacklist-attribute";
+
+    const char* PRINT_CACHE = "print-cache";
+    const char* PRINT_CACHE_BY_ENDPOINT = "print-cache-by-endpoint";
+    const char* PRINT_CACHE_TYPE = "print-cache-type";
+    const char* PRINT_CACHE_WHERE = "print-cache-where";
+}
+
 void print_usage(const std::string& program_name,
                  const po::options_description& options) {
     std::cout << "Usage: " << program_name << " [OPTIONS] <config-file>\n\n";
@@ -103,23 +117,6 @@ void write_status(const std::string& file,
     of << "}" << std::endl;
 }
 
-/**
- * Splits a string with format "variable=value" into its parts.
- * Used on the command line for overriding config file variables.
- */
-void parse_override(const std::string& override_str,
-                    std::string& variable,
-                    std::string& value) {
-    auto pos = override_str.find('=');
-
-    if (pos == std::string::npos) {
-        throw std::runtime_error("Malformed variable override (" + override_str + ")");
-    }
-
-    variable = override_str.substr(0, pos);
-    value = override_str.substr(pos+1);
-}
-
 /**
  * Modifies a set of objects so they will be considered different from what's in the data source,
  * used by --force-update.
@@ -150,7 +147,7 @@ void make_dirty(std::shared_ptr<rendered_object_list> objects, const std::vector
  * parameter will let the caller know if there was a cache file or not.
  */
 std::shared_ptr<rendered_object_list> read_cache(const post_processing::plugins& ppp, bool *cache_file_existed) {
-    auto cache_path = config_file::instance().get_path("cache-file");
+    auto cache_path = config_file::instance().get_path(options::CACHE_FILE);
 
     if (cache_path.empty()) {
         return nullptr;
@@ -187,30 +184,26 @@ std::shared_ptr<rendered_object_list> read_cache(const post_processing::plugins&
     return rendered_cache;
 }
 
-// TODO: Add the rest of the options names here
-namespace options {
-    const char* USER_BLACKLIST_FILE = "user-blacklist-file";
-    const char* USER_BLACKLIST_ATTRIBUTE = "user-blacklist-attribute";
-}
-
 int main(int argc, char *argv[]) {
     try {
         po::options_description cmdline_options("All options");
         po::options_description generic("Options");
         po::options_description hidden("Hidden options");
 
         generic.add_options()
-            ("help,h",             "produce help message")
-            ("version,v",          "displays version of this program")
-            ("rebuild-cache,r",    "ignores cache file contents and instead queries SCIM server for list of objects")
-            ("skip-load",          "don't read from data source, causes delete for all objects in cache")
-            ("skip-thresholds",    "don't verify thresholds");
+            ("help,h",                         "produce help message")
+            ("version,v",                      "displays version of this program")
+            ("rebuild-cache,r",                "ignores cache file contents and instead queries SCIM server for list of objects")
+            ("skip-load",                      "don't read from data source, causes delete for all objects in cache")
+            ("skip-thresholds",                "don't verify thresholds")
+            (options::PRINT_CACHE,             "prints contents of cache file (see --print-cache-type and --print-cache-where)")
+            (options::PRINT_CACHE_BY_ENDPOINT, "uses the SCIM endpoints instead of EGIL types when printing the cache file");
 
         // Config file variables exposed as command line options
         std::vector<config_file_option> common_vars =
             { { "cert",             "client certificate",               true },
               { "key",              "client private key",               true },
-              { "cache-file",       "cache file",                       true },
+              { options::CACHE_FILE,"cache file",                       true },
               { "metadata-path",    "metadata file",                    true },
               { "metadata-entity",  "entity in metadata to connect to", false },
               { "metadata-server",
@@ -240,6 +233,10 @@ int main(int argc, char *argv[]) {
         generic.add_options()
             (options::USER_BLACKLIST_FILE, po::value<std::string>(), "a file of users which shall be blocked from loading")
             (options::USER_BLACKLIST_ATTRIBUTE, po::value<std::string>(), "attribute (in the data source) to match against user blacklist file");
+        
+        generic.add_options()
+            (options::PRINT_CACHE_TYPE, po::value<std::vector<std::string>>(), "only print given type(s)")
+            (options::PRINT_CACHE_WHERE, po::value<std::vector<std::string>>(), "only print objects where attributes match given values");
 
         hidden.add_options()
             ("config-file", po::value<std::vector<std::string>>(), "config file");
@@ -273,7 +270,7 @@ int main(int argc, char *argv[]) {
             std::cerr << e.what() << std::endl;
             return EXIT_FAILURE;
         }
-        
+
         config_file &config = config_file::instance();
 
         std::string config_file;
@@ -287,8 +284,6 @@ int main(int argc, char *argv[]) {
         }
 
         /** Load configuration file */
-        std::cout << "processing: " << config_file << std::endl;
-
         time_t start_time = time(nullptr);
             
         int err = 0;
@@ -324,6 +319,34 @@ int main(int argc, char *argv[]) {
             }
         }
 
+        if (vm.count(options::PRINT_CACHE)) {
+            bool by_endpoint = vm.count(options::PRINT_CACHE_BY_ENDPOINT);
+            auto cache_path = config_file::instance().get_path(options::CACHE_FILE);
+
+            std::shared_ptr<rendered_object_list> cache;
+            try {
+                cache = rendered_cache_file::get_contents(cache_path);
+            }
+            catch (const rendered_cache_file::bad_format &) {
+                std::cerr << "Unrecognized cache file format" << std::endl;
+                return EXIT_FAILURE;
+            }
+            catch (const std::runtime_error &e) {
+                std::cerr << "Failed to read cache file: " << e.what() << std::endl;
+                return EXIT_FAILURE;
+            }
+
+            std::vector<std::string> types, where;
+            if (vm.count(options::PRINT_CACHE_TYPE)) {
+                types = vm[options::PRINT_CACHE_TYPE].as<std::vector<std::string>>();
+            }
+            if (vm.count(options::PRINT_CACHE_WHERE)) {
+                where = vm[options::PRINT_CACHE_WHERE].as<std::vector<std::string>>();
+            }
+            print_cache(cache, by_endpoint, types, where);
+            return EXIT_SUCCESS;
+        }
+
         add_scim_vars_for_virtual_groups();
 
         if (config.get_bool("scim-auth-WEAK")) {
diff --git a/src/print_cache.cpp b/src/print_cache.cpp
@@ -0,0 +1,125 @@
+/**
+ *  This file is part of the EGIL SCIM client.
+ *
+ *  Copyright (C) 2017-2025 Föreningen Sambruk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "print_cache.hpp"
+#include "config_file.hpp"
+#include <algorithm>
+#include <iostream>
+
+#include <boost/property_tree/json_parser.hpp>
+
+namespace pt = boost::property_tree;
+
+namespace {
+/** Checks if all where conditions matches the json.
+  * A where condition is something like externalId=foo
+  * Paths can also be used to get sub-attributes, such as name/givenName=Babs
+  * boost::property_tree paths are used with / as separator.
+  */
+bool where_matches(const std::vector<std::string> &where, const std::string& json) {
+    // No need to parse JSON if there are no where conditions
+    if (where.size() == 0) {
+        return true;
+    }
+    pt::ptree root;
+    try {
+        std::stringstream json_stream;
+        json_stream << json;
+        pt::read_json(json_stream, root);
+    }
+    catch (const std::runtime_error &) {
+        // Unparsable JSON isn't printed if we have where conditions
+        return false;
+    }
+
+    bool all_matched = true;
+    for (auto &condition : where) {
+        std::string variable, value;
+        try {
+            parse_override(condition, variable, value);
+            if (root.get<std::string>(pt::ptree::path_type{variable, '/'}) != value) {
+                all_matched = false;
+                break;
+            }
+        } catch (std::runtime_error& e) {
+            // either we couldn't parse the condition, or the object
+            // didn't have the variable
+            all_matched = false;
+            break;
+        }
+    }
+    return all_matched;
+}
+}
+
+void print_cache(std::shared_ptr<rendered_object_list> cache,
+                 bool by_endpoint,
+                 const std::vector<std::string> &types,
+                 const std::vector<std::string> &where) {
+
+    config_file &conf = config_file::instance();
+    std::map<std::string, std::vector<std::string>> to_print_per_type;
+
+    for (auto &itr : *cache) {
+        auto obj = itr.second;
+
+        auto obj_type = obj->get_type();
+
+        if (by_endpoint) {
+            auto endpoint_variable = obj_type + "-scim-url-endpoint";
+            if (conf.has(endpoint_variable)) {
+                obj_type = conf.get(endpoint_variable);
+            }
+        }
+
+        if (types.size() == 0 || std::find(types.begin(), types.end(), obj_type) != types.end()) {
+            auto json = obj->get_json();
+            if (where_matches(where, json)) {
+                to_print_per_type[obj_type].push_back(json);
+            }
+        }
+    }
+
+    std::cout << "{\n";
+
+    bool first_type = true;
+    for (auto &itr : to_print_per_type) {
+        if (!first_type) {
+            std::cout << ",\n";
+        }
+        auto& current_type = itr.first;
+        auto& current_objects = itr.second;
+
+        std::cout << "\"" << current_type << "\": [\n";
+
+        bool first_object = true;
+        for (auto &jtr : current_objects) {
+            if (!first_object) {
+                std::cout << ",\n";
+            }
+            std::cout << jtr << "\n";
+            first_object = false;
+        }
+
+        std::cout << "]";
+        first_type = false;
+    }
+
+    std::cout << "\n}" << std::endl;
+}
diff --git a/src/print_cache.hpp b/src/print_cache.hpp
@@ -0,0 +1,42 @@
+/**
+ *  This file is part of the EGIL SCIM client.
+ *
+ *  Copyright (C) 2017-2025 Föreningen Sambruk
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef EGILSCIM_PRINT_CACHE_HPP
+#define EGILSCIM_PRINT_CACHE_HPP
+
+#include <memory>
+#include <vector>
+#include <string>
+#include "model/rendered_object_list.hpp"
+
+/** Prints certain objects from the cache to stdout.
+ *  types can be used to limit the objects to certain types, if types is empty
+ *  objects of all types are printed.
+ *  If by_endpoint is true the types are interpreted as SCIM endpoints instead
+ *  of EGIL types.
+ *  where contains conditions that must be met for each object to print, in the
+ *  format attribute=value. For instance userName=babs@example.com . If there
+ *  are multiple where conditions all must be met.
+ */
+void print_cache(std::shared_ptr<rendered_object_list> cache,
+                 bool by_endpoint,
+                 const std::vector<std::string> &types,
+                 const std::vector<std::string> &where);
+
+#endif // EGILSCIM_PRINT_CACHE_HPP
diff --git a/src/utility/utils.cpp b/src/utility/utils.cpp
diff --git a/src/utility/utils.hpp b/src/utility/utils.hpp
