diff --git a/.dockerignore b/.dockerignore index 2c565ee59..b43f71e54 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,12 +18,9 @@ !dc_measurements/package.xml !dc_services/package.xml !dc_cli/package.xml -!dc_destinations/package.xml -!fluent_bit_plugins/package.xml !dc_common/package.xml !dc_interfaces/package.xml !dc_util/package.xml !dc_description/package.xml !dc_core/package.xml !dc_group/package.xml -!fluent_bit_vendor/package.xml diff --git a/.gitignore b/.gitignore index 2cde551d9..bb3982a2a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,12 +26,6 @@ msg/_*.py **build_isolated/ **devel_isolated/ -# Fluent Bit shared libraries -fluent_bit_plugins/src/go/out_minio/out_minio.so -fluent_bit_plugins/src/go/out_minio/out_minio.h -fluent_bit_plugins/src/go/out_files_metrics/out_files_metrics.so -fluent_bit_plugins/src/go/out_files_metrics/out_files_metrics.h - # Generated by dynamic reconfigure *.cfgc /cfg/cpp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 947820b9c..5f78c4de0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -157,7 +157,3 @@ repos: - id: poetry-requirements name: Check requirements-dev.txt args: [-o, requirements-dev.txt, --with, dev, --without-hashes] - - repo: https://github.com/tekwizely/pre-commit-golang - rev: v1.0.0-rc.1 - hooks: - - id: go-fmt diff --git a/CLAUDE.md b/CLAUDE.md index 1b5a1e065..448982214 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,6 @@ are already recorded there rather than in code comments. | `dc_common` | Common support functionality used throughout the DC stack | | `dc_measurements` | Collect data with Measurement plugins | | `dc_group` | Group node — merges Records from several Measurements by time proximity | -| `dc_destinations` | Send Records to Destination plugins | | `dc_services` | Collect uptime data | | `dc_lifecycle_manager` | Controller/manager for the DC system's lifecycle nodes | | `dc_interfaces` | Data collection ROS interfaces (msgs/srvs) | @@ -37,7 +36,6 @@ are already recorded there rather than in code comments. | `dc_demos` | Demo packages | | `dc_simulation` | Warehouse simulation | | `dc_util` | General shared headers | -| `fluent_bit_plugins`, `fluent_bit_vendor` | Embedded Fluent Bit (humble-era); being demolished per ADR-0001 in the `jazzy` line | ## Build / lint diff --git a/dc_bridge/test/supervisor_test.cpp b/dc_bridge/test/supervisor_test.cpp index 09487abf1..4b644c7c9 100644 --- a/dc_bridge/test/supervisor_test.cpp +++ b/dc_bridge/test/supervisor_test.cpp @@ -22,6 +22,27 @@ SupervisorConfig sh(const std::string& script, std::chrono::milliseconds backoff cfg.restart_backoff = backoff; return cfg; } + +// Polls `pred` until it's true or `timeout` elapses. A fixed sleep_for() followed by a +// single assertion assumes the supervised child (fork+exec, then whatever it runs) is +// always scheduled within that margin — a loaded/shared CI runner can stall it well +// past a couple hundred milliseconds, which made RestartsProcessThatExitsOnItsOwn and +// RespectsRestartBackoff flaky. Polling only waits as long as actually needed and +// still fails loudly if the condition is never met by the deadline. +template +bool wait_until(Pred pred, std::chrono::milliseconds timeout = std::chrono::seconds(5)) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline) + { + if (pred()) + { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return pred(); +} } // namespace TEST(Supervisor, RestartsProcessThatExitsOnItsOwn) @@ -30,8 +51,7 @@ TEST(Supervisor, RestartsProcessThatExitsOnItsOwn) s.start(); EXPECT_TRUE(s.is_running()); - std::this_thread::sleep_for(std::chrono::milliseconds(300)); - EXPECT_FALSE(s.is_running()); + EXPECT_TRUE(wait_until([&] { return !s.is_running(); })) << "child never exited"; EXPECT_TRUE(s.poll_restart()); EXPECT_TRUE(s.is_running()); @@ -41,12 +61,10 @@ TEST(Supervisor, RespectsRestartBackoff) { Supervisor s(sh("exit 1", std::chrono::seconds(60))); s.start(); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - EXPECT_FALSE(s.is_running()); + EXPECT_TRUE(wait_until([&] { return !s.is_running(); })) << "child never exited"; EXPECT_TRUE(s.poll_restart()); // first restart is immediate (no prior exit recorded) - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - EXPECT_FALSE(s.is_running()); + EXPECT_TRUE(wait_until([&] { return !s.is_running(); })) << "child never exited"; EXPECT_FALSE(s.poll_restart()); // second exit within the 60s backoff window } diff --git a/dc_bringup/launch/dc_bringup.launch.py b/dc_bringup/launch/dc_bringup.launch.py index 01a9a0d55..6d0da36c1 100644 --- a/dc_bringup/launch/dc_bringup.launch.py +++ b/dc_bringup/launch/dc_bringup.launch.py @@ -257,12 +257,10 @@ def generate_launch_description(): parameters=[configured_params], arguments=["--ros-args", "--log-level", log_level], ), - # destination_server (dc_destinations) is COLCON_IGNOREd on the jazzy line pending - # the DC 2.0 embedded-Fluent-Bit demolition slice (#241/#242); dc_bridge stands - # in its place, forwarding Records to the Vector shipper it supervises internally - # (ADRs 0001/0006/0007). It is a plain node outside the lifecycle manager, so it - # isn't in lifecycle_manager_dc's node_names; launch respawn supervises it - # unconditionally (ADR-0006), independent of use_respawn. + # dc_bridge spawns and supervises the Vector shipper and forwards Records to + # it. It's a plain node outside the lifecycle manager (not in + # lifecycle_manager_dc's node_names, see ADR-0006), so launch respawn + # supervises it unconditionally, independent of use_respawn. Node( package="dc_bridge", executable="dc_bridge", @@ -339,12 +337,9 @@ def generate_launch_description(): parameters=[configured_params], arguments=["--ros-args", "--log-level", log_level], ), - # destination_server (dc_destinations) is COLCON_IGNOREd on the jazzy line pending - # the DC 2.0 embedded-Fluent-Bit demolition slice (#241/#242); dc_bridge stands - # in its place. It isn't built/registered as an rclcpp_components plugin, so it - # always runs as a plain node (ADR-0006), even when the rest of the stack is - # composed; launch respawn supervises it unconditionally, independent of - # use_respawn. + # dc_bridge isn't built/registered as an rclcpp_components plugin, so it always + # runs as a plain node (ADR-0006), even when the rest of the stack is composed; + # launch respawn supervises it unconditionally, independent of use_respawn. Node( package="dc_bridge", executable="dc_bridge", diff --git a/dc_bringup/params/dc_params.yaml b/dc_bringup/params/dc_params.yaml index 2106bdebe..e631f465b 100644 --- a/dc_bringup/params/dc_params.yaml +++ b/dc_bringup/params/dc_params.yaml @@ -57,10 +57,10 @@ dc_bridge: # scanned for the `local_paths`/`remote_paths` File references Measurements embed # (camera, map, …), each File is uploaded (multipart + resumable for large ones) # and verified, and per-File status plus group-completion metadata Records are - # routed to `files.metadata_destination`. The destination name ("minio" here) must + # routed to `files.metadata_destination`. The destination name ("rustfs" here) must # match the key the Measurement writes under `remote_paths`. # - # minio: + # rustfs: # type: s3 # receives: files # inputs: ["/dc/measurement/camera"] diff --git a/dc_core/include/dc_core/destination.hpp b/dc_core/include/dc_core/destination.hpp deleted file mode 100644 index 7c2eb06ea..000000000 --- a/dc_core/include/dc_core/destination.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef DC_CORE_DESTINATION_HPP_ -#define DC_CORE_DESTINATION_HPP_ - -#include -#include - -#include "dc_interfaces/msg/string_stamped.hpp" -#include "pluginlib/class_loader.hpp" -#include "rclcpp/rclcpp.hpp" -#include "rclcpp_lifecycle/lifecycle_node.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_core -{ -class Destination -{ -public: - using Ptr = std::shared_ptr; - - /** - * @brief Virtual destructor - */ - virtual ~Destination() - { - } - - /** - * @param parent pointer to user's node - * @param name The name of this destination - * @param inputs The name of the inputs matching this destination - * @param debug Display debug messages - */ - virtual void configure(const rclcpp_lifecycle::LifecycleNode::WeakPtr& parent, const std::string& name, - const std::vector& inputs, flb_ctx_t* ctx, const bool& debug, - const std::string& flb_in_storage_type, const std::string& time_format, - const std::string& time_key) = 0; - - /** - * @brief Method to cleanup resources used on shutdown. - */ - virtual void cleanup() = 0; - - /** - * @brief Method to activate Destination and any threads involved in execution. - */ - virtual void activate() = 0; - - /** - * @brief Method to deactivate Destination and any threads involved in execution. - */ - virtual void deactivate() = 0; - - virtual void sendData(dc_interfaces::msg::StringStamped::SharedPtr msg) = 0; -}; -} // namespace dc_core - -#endif // DC_CORE_DESTINATION_HPP_ diff --git a/dc_destinations/CMakeLists.txt b/dc_destinations/CMakeLists.txt deleted file mode 100644 index 6e0e45fa9..000000000 --- a/dc_destinations/CMakeLists.txt +++ /dev/null @@ -1,142 +0,0 @@ -cmake_minimum_required(VERSION 3.8) -project(dc_destinations) - -set(dependencies - ament_index_cpp - dc_core - dc_interfaces - dc_util - fluent_bit_vendor - nav2_util - pluginlib - rclcpp_components - rclcpp_lifecycle -) - -find_package(ament_cmake REQUIRED) -find_package(ament_cmake_ros REQUIRED) -foreach(Dependency IN ITEMS ${dependencies}) - find_package(${Dependency} REQUIRED) -endforeach() -find_package(fluent_bit REQUIRED) - -dc_package() - -set(library_name destination_server_core) -set(executable_name destination_server) - -include_directories( - include -) - -# Plugins destinations -add_library(dc_flb_file_destination SHARED plugins/flb_file.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_file_destination) - -add_library(dc_flb_files_metrics_destination SHARED plugins/flb_files_metrics.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_files_metrics_destination) - -add_library(dc_flb_http_destination SHARED plugins/flb_http.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_http_destination) - -add_library(dc_flb_influxdb_destination SHARED plugins/flb_influxdb.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_influxdb_destination) - -add_library(dc_flb_kinesis_streams_destination SHARED plugins/flb_kinesis_streams.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_kinesis_streams_destination) - -add_library(dc_flb_minio_destination SHARED plugins/flb_minio.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_minio_destination) - -add_library(dc_flb_null_destination SHARED plugins/flb_null.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_null_destination) - -add_library(dc_flb_pgsql_destination SHARED plugins/flb_pgsql.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_pgsql_destination) - -add_library(dc_flb_s3_destination SHARED plugins/flb_s3.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_s3_destination) - -add_library(dc_flb_slack_destination SHARED plugins/flb_slack.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_slack_destination) - -add_library(dc_flb_stdout_destination SHARED plugins/flb_stdout.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_stdout_destination) - -add_library(dc_flb_tcp_destination SHARED plugins/flb_tcp.cpp) -list(APPEND dc_destination_plugin_libs dc_flb_tcp_destination) - -add_library(dc_rcl_destination SHARED plugins/rcl.cpp) -list(APPEND dc_destination_plugin_libs dc_rcl_destination) - -foreach(destination_plugin ${dc_destination_plugin_libs}) - ament_target_dependencies(${destination_plugin} ${dependencies}) - target_link_libraries( - ${destination_plugin} - fluent_bit::fluent_bit - ) - target_compile_definitions(${destination_plugin} PRIVATE BT_PLUGIN_EXPORT) -endforeach() - -pluginlib_export_plugin_description_file(dc_core destination_plugin.xml) - -# Library -add_library(${library_name} SHARED - src/destination_server.cpp -) - -target_link_libraries( - ${library_name} - fluent_bit::fluent_bit -) - -ament_target_dependencies(${library_name} - ${dependencies} -) - -# Executable -add_executable(${executable_name} - src/main.cpp -) - -target_link_libraries( - ${executable_name} - ${library_name} -) - -ament_target_dependencies(${executable_name} - ${dependencies} -) - -rclcpp_components_register_nodes(${library_name} "destination_server::DestinationServer") - - -install(TARGETS ${library_name} - ${dc_destination_plugin_libs} - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin -) - -install(TARGETS ${executable_name} - RUNTIME DESTINATION lib/${PROJECT_NAME} -) - -install(DIRECTORY include/ - DESTINATION include/ -) - -install(FILES destination_plugin.xml - DESTINATION share/${PROJECT_NAME} -) - -install(DIRECTORY plugins/ - DESTINATION share/${PROJECT_NAME}/plugins/ -) - -ament_export_include_directories(include) -ament_export_libraries(${library_name} - ${dc_destination_plugin_libs} -) -ament_export_dependencies(${dependencies}) -ament_package() diff --git a/dc_destinations/COLCON_IGNORE b/dc_destinations/COLCON_IGNORE deleted file mode 100644 index 5214efb72..000000000 --- a/dc_destinations/COLCON_IGNORE +++ /dev/null @@ -1,4 +0,0 @@ -Ignored on the jazzy line pending the DC 2.0 embedded-Fluent-Bit demolition (ADR-0003: -retire the pluginlib Destination layer in favor of the Bridge's blessed Destinations + -passthrough). See the dc-2.0 epic (#241). Do not delete this package; it is dropped, not -removed. diff --git a/dc_destinations/destination_plugin.xml b/dc_destinations/destination_plugin.xml deleted file mode 100644 index bd38726e0..000000000 --- a/dc_destinations/destination_plugin.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - dc_destination_flb_file - - - - - - - - dc_destination_flb_files_metrics - - - - - - - - dc_destination_flb_http - - - - - - - - dc_destination_flb_influxdb - - - - - - - - dc_destination_flb_kinesis_streams - - - - - - - - dc_destination_flb_minio - - - - - - - - dc_destination_flb_null - - - - - - - - dc_destination_flb_pgsql - - - - - - - - dc_destination_flb_s3 - - - - - - - - dc_destination_flb_stdout - - - - - - - - dc_destination_flb_slack - - - - - - - - dc_destination_flb_tcp - - - - - - - - dc_destination_rcl - - - - diff --git a/dc_destinations/include/dc_destinations/destination.hpp b/dc_destinations/include/dc_destinations/destination.hpp deleted file mode 100644 index 258dc547a..000000000 --- a/dc_destinations/include/dc_destinations/destination.hpp +++ /dev/null @@ -1,175 +0,0 @@ -#ifndef DC_DESTINATIONS__DESTINATION_HPP_ -#define DC_DESTINATIONS__DESTINATION_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dc_core/destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" -#include "rclcpp/rclcpp.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -using namespace std::chrono_literals; // NOLINT -using json = nlohmann::json; - -/** - * @class nav2_behaviors::Behavior - * @brief An action server Behavior base class implementing the action server and basic factory. - */ -// template -class Destination : public dc_core::Destination -{ -public: - // using ActionServer = nav2_util::SimpleActionServer; - - /** - * @brief A Destination constructor - */ - Destination() - { - } - - virtual ~Destination() = default; - - // an opportunity for derived classes to do something on configuration - // if they chose - virtual void onConfigure() - { - } - - // an opportunity for derived classes to do something on configuration - // if they chose - virtual void onDestinationConfigure() - { - } - - // an opportunity for derived classes to do something on cleanup - // if they chose - virtual void onCleanup() - { - } - - std::shared_ptr getNode() - { - auto node = node_.lock(); - if (!node) - { - throw std::runtime_error{ "Failed to lock node" }; - } - return node; - } - - virtual void sendData(dc_interfaces::msg::StringStamped::SharedPtr msg) - { - RCLCPP_DEBUG(logger_, "msg: %s", msg->data.c_str()); - } - - void dataCb(dc_interfaces::msg::StringStamped::SharedPtr msg) - { - sendData(msg); - } - - // configure the server on lifecycle setup - void configure(const rclcpp_lifecycle::LifecycleNode::WeakPtr& parent, const std::string& name, - const std::vector& inputs, flb_ctx_t* ctx, const bool& debug, - const std::string& flb_in_storage_type, const std::string& time_format, - const std::string& time_key) override - { - node_ = parent; - auto node = node_.lock(); - - inputs_ = inputs; - debug_ = debug; - ctx_ = ctx; - flb_in_storage_type_ = flb_in_storage_type; - time_format_ = time_format; - time_key_ = time_key; - - logger_ = node->get_logger(); - - for (auto& input : inputs_) - { - subscriptions_.push_back(node->create_subscription( - input.c_str(), 10, std::bind(&Destination::dataCb, this, std::placeholders::_1))); - } - - RCLCPP_INFO(logger_, "Configuring %s", name.c_str()); - - destination_name_ = name; - - onDestinationConfigure(); - onConfigure(); - - RCLCPP_INFO(logger_, "Done configuring %s", destination_name_.c_str()); - } - - // Cleanup server on lifecycle transition - void cleanup() override - { - onCleanup(); - } - - // Activate server on lifecycle transition - void activate() override - { - RCLCPP_INFO(logger_, "Activating destination %s", destination_name_.c_str()); - - enabled_ = true; - } - - // Deactivate server on lifecycle transition - void deactivate() override - { - enabled_ = false; - } - -protected: - rclcpp_lifecycle::LifecycleNode::WeakPtr node_; - - std::string destination_name_; - std::vector inputs_; - bool enabled_; - - // Clock - rclcpp::Clock steady_clock_{ RCL_STEADY_TIME }; - - // Timer - rclcpp::TimerBase::SharedPtr collect_timer_; - std::vector::SharedPtr> subscriptions_; - - // Logger - rclcpp::Logger logger_{ rclcpp::get_logger("dc_destinations") }; - - bool debug_{ false }; - - // Fluent Bit - flb_ctx_t* ctx_; - std::string flb_in_storage_type_; - std::string time_format_{ "iso8601" }; - std::string time_key_{ "date" }; - - // CB - rclcpp::CallbackGroup::SharedPtr timer_cb_group_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__DESTINATION_HPP_ diff --git a/dc_destinations/include/dc_destinations/destination_server.hpp b/dc_destinations/include/dc_destinations/destination_server.hpp deleted file mode 100644 index db7f9b591..000000000 --- a/dc_destinations/include/dc_destinations/destination_server.hpp +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef DC_DESTINATIONS__DESTINATION_SERVER_HPP_ -#define DC_DESTINATIONS__DESTINATION_SERVER_HPP_ - -#include -#include -#include -#include -#include -#include - -#include "dc_core/destination.hpp" -#include "dc_util/filesystem_utils.hpp" -#include "dc_util/node_utils.hpp" -#include "dc_util/string_utils.hpp" -#include "nav2_util/lifecycle_node.hpp" -#include "pluginlib/class_list_macros.hpp" -#include "pluginlib/class_loader.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace destination_server -{ -using namespace std::chrono_literals; - -/** - * @class dc_destination::DestinationNode - * @brief An server hosting a map of destination plugins - */ -class DestinationServer : public nav2_util::LifecycleNode -{ -public: - /** - * @brief A constructor for dc_destination::DestinationNode - * @param options Additional options to control creation of the node. - */ - explicit DestinationServer(const rclcpp::NodeOptions& options = rclcpp::NodeOptions()); - ~DestinationServer() override; - - /** - * @brief Loads destination plugins from parameter file - * @return bool if successfully loaded the plugins - */ - bool loadDestinationPlugins(); - -protected: - /** - * @brief Configure lifecycle server - */ - nav2_util::CallbackReturn on_configure(const rclcpp_lifecycle::State& state) override; - - /** - * @brief Activate lifecycle server - */ - nav2_util::CallbackReturn on_activate(const rclcpp_lifecycle::State& state) override; - - /** - * @brief Deactivate lifecycle server - */ - nav2_util::CallbackReturn on_deactivate(const rclcpp_lifecycle::State& state) override; - - /** - * @brief Cleanup lifecycle server - */ - nav2_util::CallbackReturn on_cleanup(const rclcpp_lifecycle::State& state) override; - - /** - * @brief Shutdown lifecycle server - */ - nav2_util::CallbackReturn on_shutdown(const rclcpp_lifecycle::State& state) override; - - // Plugins - pluginlib::ClassLoader plugin_loader_; - std::vector> destinations_; - std::vector default_ids_; - - std::vector destination_ids_; - std::vector ros_topics_; - std::vector> destination_inputs_; - std::vector destination_debug_; - std::vector destination_types_; - std::vector destination_time_format_; - std::vector destination_time_key_; - - // Fluent Bit - flb_ctx_t* ctx_; - void initFlb(); - void initFlbInputPlugin(); - void startFlbEngine(); - int flb_flush_, flb_grace_, flb_scheduler_cap_, flb_scheduler_base_; - std::string flb_log_level_, flb_storage_path_, flb_storage_sync_, flb_storage_checksum_, - flb_storage_backlog_mem_limit_, flb_input_tag_, flb_in_storage_type_, flb_in_storage_pause_on_chunks_overlimit_; - bool flb_http_server_; - std::string flb_http_listen_; - int flb_http_port_; - std::string ros2_plugin_path_; - std::string ros2_plugin_path_default_; - int ros2_plugin_spin_time_ms_; -}; - -} // namespace destination_server - -#endif // DC_DESTINATIONS__DESTINATION_SERVER_HPP_ diff --git a/dc_destinations/include/dc_destinations/flb_destination.hpp b/dc_destinations/include/dc_destinations/flb_destination.hpp deleted file mode 100644 index f3d36470d..000000000 --- a/dc_destinations/include/dc_destinations/flb_destination.hpp +++ /dev/null @@ -1,260 +0,0 @@ -#ifndef DC_DESTINATIONS__FLB_DESTINATION_HPP_ -#define DC_DESTINATIONS__FLB_DESTINATION_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dc_destinations/destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" -#include "dc_util/node_utils.hpp" -#include "dc_util/string_utils.hpp" -#include "nav2_util/node_utils.hpp" -#include "rclcpp/rclcpp.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -using namespace std::chrono_literals; // NOLINT - -class FlbDestination : public dc_destinations::Destination -{ -public: - /** - * @brief A Destination constructor - */ - FlbDestination() - { - } - - virtual void initFlbOutputPlugin() = 0; - - // configure the server on lifecycle setup - void configure(const rclcpp_lifecycle::LifecycleNode::WeakPtr& parent, const std::string& name, - const std::vector& inputs, flb_ctx_t* ctx, const bool& debug, - const std::string& flb_in_storage_type, const std::string& time_format, - const std::string& time_key) override - { - node_ = parent; - auto node = node_.lock(); - - ctx_ = ctx; - debug_ = debug; - destination_name_ = name; - inputs_ = inputs; - flb_in_storage_type_ = flb_in_storage_type; - time_format_ = time_format; - time_key_ = time_key; - - logger_ = node->get_logger(); - - RCLCPP_INFO(logger_, "Configuring Flb plugin %s", destination_name_.c_str()); - - onConfigure(); - onDestinationConfigure(); - - RCLCPP_INFO(logger_, "Done configuring Flb plugin %s", destination_name_.c_str()); - } - - ~FlbDestination() - { - } - - std::string formatString(const dc_interfaces::msg::StringStamped::SharedPtr msg) - { - std::string data_str = msg->data.c_str(); - boost::replace_all(data_str, "'", "\""); - json data_json = json::parse(data_str); - - data_json["date"] = - std::stod(std::to_string(msg->header.stamp.sec) + std::string(".") + std::to_string(msg->header.stamp.nanosec)); - - std::string formatted_str = nlohmann::to_string(data_json); - boost::replace_all(formatted_str, "'", "\""); - return std::string("[") + std::to_string(msg->header.stamp.sec) + std::string(".") + - std::to_string(msg->header.stamp.nanosec) + "," + formatted_str + std::string("]"); - } - - void initTimestampFilter() - { - /* Filter for timestamp*/ - int f_ffd = flb_filter(ctx_, (char*)"lua", NULL); - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start lua timestamp filter"); - } - std::string date_record = ""; - if (!time_key_.empty()) - { - if (time_format_ == "double") - { - date_record = "record[\"" + time_key_ + "\"] = timestamp "; - } - else if (time_format_ == "iso8601") - { - date_record = "record[\"" + time_key_ + - "\"] = os.date('!%Y-%m-%dT%H:%M:%S', timestamp[\"sec\"]) .. \".\" .. " - "tostring(math.floor(timestamp[\"nsec\"])) "; - } - } - std::string ts_lua_code = - std::string("function replace_ts(tag, timestamp, record) ") + date_record + " return 2, timestamp, record end"; - int ret = flb_filter_set(ctx_, f_ffd, "code", ts_lua_code.c_str(), NULL); - ret += flb_filter_set(ctx_, f_ffd, "call", "replace_ts", NULL); - ret += flb_filter_set(ctx_, f_ffd, "Match", destination_name_.c_str(), NULL); - if (time_format_ != "double") - { - ret += flb_filter_set(ctx_, f_ffd, "time_as_table", "true", NULL); - } - if (ret != 0) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot set lua timestamp filter"); - } - - RCLCPP_INFO(logger_, "Loaded timestamp lua filter. Match=%s, code=%s", destination_name_.c_str(), - ts_lua_code.c_str()); - } - - void initRewriteTagFilter() - { - int f_ffd = flb_filter(ctx_, (char*)"rewrite_tag", NULL); - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start rewrite_tag filter"); - } - std::string rule = "$tags .*(" + destination_name_ + ").* " + destination_name_ + " true"; - - int ret = flb_filter_set(ctx_, f_ffd, "Rule", rule.c_str(), NULL); - ret += flb_filter_set(ctx_, f_ffd, "Match", "ros2", NULL); - ret += flb_filter_set(ctx_, f_ffd, "emitter_storage.type", flb_in_storage_type_.c_str(), NULL); - ret += flb_filter_set(ctx_, f_ffd, "emitter_mem_buf_limit", "5M", NULL); - if (ret != 0) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot set rule for rewrite_tag filter"); - } - - RCLCPP_INFO(logger_, "Loaded rewrite_tag filter. Match=ros2, Rule=%s", rule.c_str()); - } - - void initConcatenateTags() - { - /* Filter rewrite tags as string configuration */ - /* ["tag1", "tag2"] -> "tag1,tag2" */ - /* and add timestamp in the field - Not sure this is needed */ - int f_ffd = flb_filter(ctx_, (char*)"lua", NULL); - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start lua filter"); - } - - std::string lua_code = std::string("function concatenate(tag, timestamp, record) ") + - "if (type(record[\"tags\"]) == \"table\") then " + - "record[\"tags\"] = table.concat(record[\"tags\"], \",\") " + "end " + - " return 2, timestamp, record end"; - - int ret = flb_filter_set(ctx_, f_ffd, "code", lua_code.c_str(), NULL); - ret += flb_filter_set(ctx_, f_ffd, "call", "concatenate", NULL); - ret += flb_filter_set(ctx_, f_ffd, "Match", "ros2", NULL); - - if (ret != 0) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot set lua filter"); - } - RCLCPP_INFO(logger_, "Loaded tags concatenation lua filter. Match=ros2, code=%s", lua_code.c_str()); - } - - void initModifyFilter() - { - /* Filter modify configuration */ - // Remove the tag from the json message - int f_ffd = flb_filter(ctx_, (char*)"modify", NULL); - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start modify filter"); - } - - int ret = flb_filter_set(ctx_, f_ffd, "Remove", "tags", NULL); - ret += flb_filter_set(ctx_, f_ffd, "Match", destination_name_.c_str(), NULL); - - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start modify filter (add custom params)"); - } - if (ret != 0) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot set rule for modify filter (add custom params)"); - } - } - - void initFlbFilters() - { - initTimestampFilter(); - - initConcatenateTags(); - - initRewriteTagFilter(); - - initModifyFilter(); - } - - void initFlbDebug() - { - /* Enable fluent bit debug */ - if (debug_) - { - int f_ffd = flb_output(ctx_, (char*)"stdout", NULL); - int ret = flb_output_set(ctx_, f_ffd, "Match", destination_name_.c_str(), NULL); - if (f_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start stdout filter"); - } - if (ret != 0) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot enable debug for plugin"); - } - } - } - - void onDestinationConfigure() override - { - initFlbFilters(); - initFlbDebug(); - initFlbOutputPlugin(); - } - -protected: - int in_ffd_; - int out_ffd_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__FLB_DESTINATION_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_file.hpp b/dc_destinations/include/dc_destinations/plugins/flb_file.hpp deleted file mode 100644 index f5f6e662c..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_file.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILE_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILE_HPP_ - -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" - -namespace dc_destinations -{ - -class FlbFile : public dc_destinations::FlbDestination -{ -public: - FlbFile(); - ~FlbFile() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string path_; - std::string file_; - std::string format_; - std::string delimiter_; - std::string label_delimiter_; - std::string template_; - bool mkdir_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILE_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_files_metrics.hpp b/dc_destinations/include/dc_destinations/plugins/flb_files_metrics.hpp deleted file mode 100644 index b4bc15920..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_files_metrics.hpp +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILESMETRICS_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILESMETRICS_HPP_ - -#include -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" -#include "dc_util/string_utils.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -// #include -#include -#pragma GCC diagnostic pop - -#define MAX_USERID_LENGTH 32 - -namespace dc_destinations -{ - -class FlbFilesMetrics : public dc_destinations::FlbDestination -{ -public: - FlbFilesMetrics(); - ~FlbFilesMetrics() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string plugin_path_; - std::string plugin_path_default_; - std::string db_type_; - std::vector file_storage_; - bool delete_when_sent_; - - std::string minio_endpoint_; - std::string minio_access_key_id_; - std::string minio_secret_access_key_; - bool minio_use_ssl_; - std::string minio_create_bucket_; - std::string minio_bucket_; - std::vector minio_upload_fields_; - std::vector minio_src_fields_; - std::vector minio_groups_; - - std::string s3_endpoint_; - std::string s3_access_key_id_; - std::string s3_secret_access_key_; - std::string s3_create_bucket_; - std::string s3_bucket_; - std::vector s3_upload_fields_; - std::vector s3_src_fields_; - std::vector s3_groups_; - - std::string pgsql_host_; - std::string pgsql_port_; - std::string pgsql_user_; - std::string pgsql_password_; - std::string pgsql_database_; - std::string pgsql_table_; - std::string pgsql_time_key_; - std::string pgsql_async_; - bool pgsql_use_ssl_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_FILESMETRICS_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_http.hpp b/dc_destinations/include/dc_destinations/plugins/flb_http.hpp deleted file mode 100644 index dae9e86a4..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_http.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_HTTP_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_HTTP_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbHTTP : public dc_destinations::FlbDestination -{ -public: - FlbHTTP(); - ~FlbHTTP() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string format_; - std::string host_; - std::string body_key_; - int port_; - std::string uri_; - std::string header_; - std::string headers_key_; - std::string header_tag_; - std::string json_date_key_; - std::string http_user_; - std::string http_passwd_; - bool log_response_payload_; - bool allow_duplicated_headers_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_HTTP_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_influxdb.hpp b/dc_destinations/include/dc_destinations/plugins/flb_influxdb.hpp deleted file mode 100644 index 3e1525a9d..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_influxdb.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_INFLUXDB_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_INFLUXDB_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbInfluxDB : public dc_destinations::FlbDestination -{ -public: - FlbInfluxDB(); - ~FlbInfluxDB() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string host_; - int port_; - std::string database_; - std::string bucket_; - std::string org_; - std::string sequence_tag_; - std::string http_user_; - std::string http_password_; - std::string http_token_; - std::vector tag_keys_; - bool auto_tags_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_INFLUXDB_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_kinesis_streams.hpp b/dc_destinations/include/dc_destinations/plugins/flb_kinesis_streams.hpp deleted file mode 100644 index 2945f8845..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_kinesis_streams.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_KINESIS_STREAMS_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_KINESIS_STREAMS_HPP_ - -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" - -namespace dc_destinations -{ - -class FlbKinesisStreams : public dc_destinations::FlbDestination -{ -public: - FlbKinesisStreams(); - ~FlbKinesisStreams() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string region_; - std::string stream_; - std::string role_arn_; - std::string time_key_; - std::string time_key_format_; - std::string log_key_; - std::string endpoint_; - bool auto_retry_requests_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_KINESIS_STREAMS_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_minio.hpp b/dc_destinations/include/dc_destinations/plugins/flb_minio.hpp deleted file mode 100644 index df6e5fb05..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_minio.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_MINIO_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_MINIO_HPP_ - -#include -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" -#include "dc_util/string_utils.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -// #include -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbMinIO : public dc_destinations::FlbDestination -{ -public: - FlbMinIO(); - ~FlbMinIO() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string endpoint_; - std::string access_key_id_; - std::string secret_access_key_; - bool use_ssl_; - bool create_bucket_; - std::string bucket_; - std::string plugin_path_; - std::string plugin_path_default_; - std::vector upload_fields_; - std::vector src_fields_; - std::vector groups_; - bool verbose_plugin_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_MINIO_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_null.hpp b/dc_destinations/include/dc_destinations/plugins/flb_null.hpp deleted file mode 100644 index 33628d05e..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_null.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_NULL_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_NULL_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbNull : public dc_destinations::FlbDestination -{ -public: - FlbNull(); - ~FlbNull() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_NULL_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_pgsql.hpp b/dc_destinations/include/dc_destinations/plugins/flb_pgsql.hpp deleted file mode 100644 index e3b0c8c8d..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_pgsql.hpp +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_PGSQL_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_PGSQL_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" -#include "dc_util/string_utils.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -#define MAX_USERID_LENGTH 32 - -namespace dc_destinations -{ - -class FlbPgSQL : public dc_destinations::FlbDestination -{ -public: - FlbPgSQL(); - ~FlbPgSQL() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string host_; - int port_; - std::string user_; - std::string password_; - std::string database_; - std::string table_; - std::string timestamp_key_; - bool async_; - std::string min_pool_size_; - std::string max_pool_size_; - bool cockroachdb_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_PGSQL_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_s3.hpp b/dc_destinations/include/dc_destinations/plugins/flb_s3.hpp deleted file mode 100644 index 9029ff2cf..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_s3.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_S3_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_S3_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbS3 : public dc_destinations::FlbDestination -{ -public: - FlbS3(); - ~FlbS3() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string region_; - std::string bucket_; - std::string json_date_key_; - std::string json_date_format_; - std::string total_file_size_; - std::string upload_chunk_size_; - std::string upload_timeout_; - std::string store_dir_; - int store_dir_limit_size_; - std::string s3_key_format_; - std::string s3_key_format_tag_delimiters_; - bool static_file_path_; - bool use_put_object_; - std::string role_arn_; - std::string endpoint_; - std::string sts_endpoint_; - std::string canned_acl_; - std::string compression_; - std::string content_type_; - bool send_content_md5_; - bool auto_retry_requests_; - std::string log_key_; - bool preserve_data_ordering_; - std::string storage_class_; - std::string retry_limit_; - std::string external_id_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_S3_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_slack.hpp b/dc_destinations/include/dc_destinations/plugins/flb_slack.hpp deleted file mode 100644 index f18c60499..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_slack.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_SLACK_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_SLACK_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbSlack : public dc_destinations::FlbDestination -{ -public: - FlbSlack(); - ~FlbSlack() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string webhook_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_SLACK_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_stdout.hpp b/dc_destinations/include/dc_destinations/plugins/flb_stdout.hpp deleted file mode 100644 index 3c0ba8203..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_stdout.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_STDOUT_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_STDOUT_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbStdout : public dc_destinations::FlbDestination -{ -public: - FlbStdout(); - ~FlbStdout() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string format_; - std::string json_date_key_; - std::string json_date_format_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_STDOUT_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/flb_tcp.hpp b/dc_destinations/include/dc_destinations/plugins/flb_tcp.hpp deleted file mode 100644 index ef2d997ba..000000000 --- a/dc_destinations/include/dc_destinations/plugins/flb_tcp.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_TCP_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_TCP_HPP_ - -#include -#include -#include -#include - -#include "dc_destinations/destination_server.hpp" -#include "dc_destinations/flb_destination.hpp" -#include "dc_interfaces/msg/string_stamped.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wparentheses" -#pragma GCC diagnostic ignored "-Wsign-compare" -#include -#pragma GCC diagnostic pop - -namespace dc_destinations -{ - -class FlbTCP : public dc_destinations::FlbDestination -{ -public: - FlbTCP(); - ~FlbTCP() override; - -protected: - /** - * @brief Configuration of behavior action - */ - void initFlbOutputPlugin() override; - void onConfigure() override; - - std::string host_; - int port_; - std::string format_; - std::string json_date_key_; - std::string json_date_format_; - int workers_; - bool tls_active_; - bool tls_verify_; - int tls_debug_; - std::string tls_ca_file_; - std::string tls_crt_file_; - std::string tls_key_file_; - std::string tls_key_passwd_; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__FLB_TCP_HPP_ diff --git a/dc_destinations/include/dc_destinations/plugins/rcl.hpp b/dc_destinations/include/dc_destinations/plugins/rcl.hpp deleted file mode 100644 index 5e7c00023..000000000 --- a/dc_destinations/include/dc_destinations/plugins/rcl.hpp +++ /dev/null @@ -1,31 +0,0 @@ - -#ifndef DC_DESTINATIONS__PLUGINS__DESTINATIONS__RCL_HPP_ -#define DC_DESTINATIONS__PLUGINS__DESTINATIONS__RCL_HPP_ - -#include -#include -#include - -#include "dc_core/destination.hpp" -#include "dc_destinations/destination.hpp" -#include "dc_destinations/destination_server.hpp" - -namespace dc_destinations -{ - -class Rcl : public dc_destinations::Destination -{ -public: - Rcl(); - ~Rcl() override; - -protected: - /** - * @brief Configuration of destination - */ - void sendData(dc_interfaces::msg::StringStamped::SharedPtr msg) override; -}; - -} // namespace dc_destinations - -#endif // DC_DESTINATIONS__PLUGINS__DESTINATIONS__RCL_HPP_ diff --git a/dc_destinations/package.xml b/dc_destinations/package.xml deleted file mode 100644 index c1aa5d1ab..000000000 --- a/dc_destinations/package.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - dc_destinations - 0.1.0 - Send all data to destination plugins - David Bensoussan - MPL-2.0 - - ament_cmake_ros - - ament_index_cpp - dc_core - dc_interfaces - dc_lifecycle_manager - dc_util - fluent_bit_vendor - nav2_util - nlohmann-json-dev - pluginlib - rclcpp - - fluent_bit_plugins - systemd - - - ament_cmake - - - diff --git a/dc_destinations/plugins/flb_file.cpp b/dc_destinations/plugins/flb_file.cpp deleted file mode 100644 index e4abeb5c9..000000000 --- a/dc_destinations/plugins/flb_file.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include "dc_destinations/plugins/flb_file.hpp" - -namespace dc_destinations -{ - -FlbFile::FlbFile() : dc_destinations::FlbDestination() -{ -} - -void FlbFile::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".path", rclcpp::ParameterValue("$HOME/data")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".file", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".format", rclcpp::ParameterValue("out_file")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".mkdir", rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".template", rclcpp::ParameterValue("")); - - node->get_parameter(destination_name_ + ".path", path_); - path_ = dc_util::expand_env(path_); - node->get_parameter(destination_name_ + ".file", file_); - node->get_parameter(destination_name_ + ".format", format_); - node->get_parameter(destination_name_ + ".mkdir", mkdir_); - node->get_parameter(destination_name_ + ".template", template_); - - std::string default_delimiter = ""; - if (format_ == "ltsv") - { - default_delimiter = "\t"; - } - else if (format_ == "csv") - { - default_delimiter = ","; - } - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".delimiter", - rclcpp::ParameterValue(default_delimiter)); - node->get_parameter(destination_name_ + ".delimiter", delimiter_); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".label_delimiter", - rclcpp::ParameterValue(".")); - node->get_parameter(destination_name_ + ".label_delimiter", label_delimiter_); -} - -void FlbFile::initFlbOutputPlugin() -{ - /* Enable output plugin 'stdout' (print records to the standard output) */ - out_ffd_ = flb_output(ctx_, "file", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "path", path_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "file", file_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "format", format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "mkdir", dc_util::boolToString(mkdir_), NULL); - flb_output_set(ctx_, out_ffd_, "template", template_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "delimiter", delimiter_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "label_delimiter", label_delimiter_.c_str(), NULL); - - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output file plugin"); - } -} - -FlbFile::~FlbFile() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbFile, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_files_metrics.cpp b/dc_destinations/plugins/flb_files_metrics.cpp deleted file mode 100644 index bd10b585f..000000000 --- a/dc_destinations/plugins/flb_files_metrics.cpp +++ /dev/null @@ -1,164 +0,0 @@ -#include "dc_destinations/plugins/flb_files_metrics.hpp" - -namespace dc_destinations -{ - -FlbFilesMetrics::FlbFilesMetrics() : dc_destinations::FlbDestination() -{ -} - -void FlbFilesMetrics::onConfigure() -{ - auto node = getNode(); - - std::string package_directory = ament_index_cpp::get_package_prefix("fluent_bit_plugins"); - plugin_path_default_ = package_directory + "/lib/out_files_metrics.so"; - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".plugin_path", - rclcpp::ParameterValue(plugin_path_default_)); - node->get_parameter(destination_name_ + ".plugin_path", plugin_path_); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".file_storage", - rclcpp::PARAMETER_STRING_ARRAY); - node->get_parameter(destination_name_ + ".file_storage", file_storage_); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".db_type", rclcpp::ParameterValue("pgsql")); - node->get_parameter(destination_name_ + ".db_type", db_type_); - node->get_parameter(destination_name_ + ".delete_when_sent", delete_when_sent_); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".delete_when_sent", - rclcpp::ParameterValue(true)); - node->get_parameter(destination_name_ + ".delete_when_sent", delete_when_sent_); - - if (std::find(file_storage_.begin(), file_storage_.end(), "minio") != file_storage_.end()) - { - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.endpoint", - rclcpp::ParameterValue("127.0.0.1:9000")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.access_key_id", - rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.secret_access_key", - rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.use_ssl", - rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.bucket", - rclcpp::ParameterValue("dc_bucket")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.upload_fields", - rclcpp::PARAMETER_STRING_ARRAY); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".minio.src_fields", - rclcpp::PARAMETER_STRING_ARRAY); - - node->get_parameter(destination_name_ + ".minio.endpoint", minio_endpoint_); - node->get_parameter(destination_name_ + ".minio.access_key_id", minio_access_key_id_); - node->get_parameter(destination_name_ + ".minio.secret_access_key", minio_secret_access_key_); - node->get_parameter(destination_name_ + ".minio.use_ssl", minio_use_ssl_); - node->get_parameter(destination_name_ + ".minio.bucket", minio_bucket_); - node->get_parameter(destination_name_ + ".minio.upload_fields", minio_upload_fields_); - node->get_parameter(destination_name_ + ".minio.src_fields", minio_src_fields_); - } - if (std::find(file_storage_.begin(), file_storage_.end(), "s3") != file_storage_.end()) - { - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.endpoint", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.access_key_id", - rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.secret_access_key", - rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.bucket", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.upload_fields", - rclcpp::PARAMETER_STRING_ARRAY); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3.src_fields", - rclcpp::PARAMETER_STRING_ARRAY); - - node->get_parameter(destination_name_ + ".s3.endpoint", s3_endpoint_); - node->get_parameter(destination_name_ + ".s3.access_key_id", s3_access_key_id_); - node->get_parameter(destination_name_ + ".s3.secret_access_key", s3_secret_access_key_); - node->get_parameter(destination_name_ + ".s3.bucket", s3_bucket_); - node->get_parameter(destination_name_ + ".s3.upload_fields", s3_upload_fields_); - node->get_parameter(destination_name_ + ".s3.src_fields", s3_src_fields_); - } - - if (db_type_ == "pgsql") - { - char username[MAX_USERID_LENGTH]; - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.host", - rclcpp::ParameterValue("127.0.0.1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.port", - rclcpp::ParameterValue("5432")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.user", - rclcpp::ParameterValue(cuserid(username))); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.password", - rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.database", - rclcpp::ParameterValue(cuserid(username))); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.table", - rclcpp::ParameterValue("pg_table")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".pgsql.ssl", rclcpp::ParameterValue(false)); - node->get_parameter(destination_name_ + ".pgsql.host", pgsql_host_); - node->get_parameter(destination_name_ + ".pgsql.port", pgsql_port_); - node->get_parameter(destination_name_ + ".pgsql.user", pgsql_user_); - node->get_parameter(destination_name_ + ".pgsql.password", pgsql_password_); - node->get_parameter(destination_name_ + ".pgsql.database", pgsql_database_); - node->get_parameter(destination_name_ + ".pgsql.table", pgsql_table_); - node->get_parameter(destination_name_ + ".pgsql.ssl", pgsql_use_ssl_); - } -} - -void FlbFilesMetrics::initFlbOutputPlugin() -{ - RCLCPP_INFO(logger_, "Loading flb_metrics as GO proxy output %s", plugin_path_.c_str()); - if (flb_plugin_load_router(strdup(plugin_path_.c_str()), ctx_->config) != 0) - { - flb_error("[plugin] error loading GO proxy plugin: %s", plugin_path_.c_str()); - throw std::runtime_error("Cannot load plugin"); - } - - out_ffd_ = flb_output(ctx_, "files_metrics", nullptr); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), nullptr); - - flb_output_set(ctx_, out_ffd_, "retry_limit", "false", nullptr); - flb_output_set(ctx_, out_ffd_, "file_storage", dc_util::to_space_separated_string(file_storage_).c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "db_type", db_type_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "delete_when_sent", dc_util::boolToString(delete_when_sent_), nullptr); - - if (std::find(file_storage_.begin(), file_storage_.end(), "minio") != file_storage_.end()) - { - flb_output_set(ctx_, out_ffd_, "minio_endpoint", minio_endpoint_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_access_key_id", minio_access_key_id_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_secret_access_key", minio_secret_access_key_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_use_ssl", dc_util::boolToString(minio_use_ssl_), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_create_bucket", minio_create_bucket_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_bucket", minio_bucket_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "minio_src_fields", dc_util::to_space_separated_string(minio_src_fields_).c_str(), - nullptr); - flb_output_set(ctx_, out_ffd_, "minio_upload_fields", - dc_util::to_space_separated_string(minio_upload_fields_).c_str(), nullptr); - } - - if (std::find(file_storage_.begin(), file_storage_.end(), "s3") != file_storage_.end()) - { - flb_output_set(ctx_, out_ffd_, "s3_endpoint", s3_endpoint_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_access_key_id", s3_access_key_id_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_secret_access_key", s3_secret_access_key_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_create_bucket", s3_create_bucket_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_bucket", s3_bucket_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_src_fields", dc_util::to_space_separated_string(s3_src_fields_).c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "s3_upload_fields", dc_util::to_space_separated_string(s3_upload_fields_).c_str(), - nullptr); - } - - if (db_type_ == "pgsql") - { - std::string pgsql_use_ssl_str = pgsql_use_ssl_ ? "require" : "disable"; - flb_output_set(ctx_, out_ffd_, "pgsql_host", pgsql_host_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_port", pgsql_port_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_user", pgsql_user_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_password", pgsql_password_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_database", pgsql_database_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_table", pgsql_table_.c_str(), nullptr); - flb_output_set(ctx_, out_ffd_, "pgsql_use_ssl", pgsql_use_ssl_str.c_str(), nullptr); - } -} - -FlbFilesMetrics::~FlbFilesMetrics() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbFilesMetrics, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_http.cpp b/dc_destinations/plugins/flb_http.cpp deleted file mode 100644 index 5ad410d1d..000000000 --- a/dc_destinations/plugins/flb_http.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "dc_destinations/plugins/flb_http.hpp" - -namespace dc_destinations -{ - -FlbHTTP::FlbHTTP() : dc_destinations::FlbDestination() -{ -} - -void FlbHTTP::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".host", rclcpp::ParameterValue("127.0.0.1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".port", rclcpp::ParameterValue(80)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".uri", rclcpp::ParameterValue("/")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".format", rclcpp::ParameterValue("json")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".header", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".headers_key", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".header_tag", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_key", - rclcpp::ParameterValue("date")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_format", - rclcpp::ParameterValue("double")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".http_user", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".http_passwd", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".log_response_payload", - rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".allow_duplicated_headers", - rclcpp::ParameterValue(true)); - node->get_parameter(destination_name_ + ".host", host_); - node->get_parameter(destination_name_ + ".body_key", body_key_); - node->get_parameter(destination_name_ + ".port", port_); - node->get_parameter(destination_name_ + ".uri", uri_); - node->get_parameter(destination_name_ + ".format", format_); - node->get_parameter(destination_name_ + ".header", header_); - node->get_parameter(destination_name_ + ".headers_key", headers_key_); - node->get_parameter(destination_name_ + ".header_tag", header_tag_); - node->get_parameter(destination_name_ + ".json_date_key", json_date_key_); - node->get_parameter(destination_name_ + ".http_user", http_user_); - node->get_parameter(destination_name_ + ".http_passwd", http_passwd_); - node->get_parameter(destination_name_ + ".log_response_payload", log_response_payload_); - node->get_parameter(destination_name_ + ".allow_duplicated_headers", allow_duplicated_headers_); -} - -void FlbHTTP::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "http", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "body_key", body_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "host", host_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "port", std::to_string(port_).c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "uri", uri_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "format", format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "header", header_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "headers_key", headers_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "header_tag", header_tag_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_key", json_date_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "http_user", http_user_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "http_passwd", http_passwd_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "log_response_payload", dc_util::boolToString(log_response_payload_), NULL); - flb_output_set(ctx_, out_ffd_, "allow_duplicated_headers", dc_util::boolToString(allow_duplicated_headers_), NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output http plugin"); - } -} - -FlbHTTP::~FlbHTTP() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbHTTP, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_influxdb.cpp b/dc_destinations/plugins/flb_influxdb.cpp deleted file mode 100644 index 43db306b8..000000000 --- a/dc_destinations/plugins/flb_influxdb.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "dc_destinations/plugins/flb_influxdb.hpp" - -namespace dc_destinations -{ - -FlbInfluxDB::FlbInfluxDB() : dc_destinations::FlbDestination() -{ -} - -void FlbInfluxDB::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".host", rclcpp::ParameterValue("127.0.0.1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".port", rclcpp::ParameterValue(8086)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".database", - rclcpp::ParameterValue("fluentbit")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".bucket", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".org", rclcpp::ParameterValue("fluent")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".sequence_tag", - rclcpp::ParameterValue("_seq")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".http_user", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".http_password", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".http_token", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tag_keys", - rclcpp::ParameterValue(std::vector())); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".auto_tags", rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tags_list_enabled", - rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tags_list_key", - rclcpp::ParameterValue("tags")); - node->get_parameter(destination_name_ + ".host", host_); - node->get_parameter(destination_name_ + ".port", port_); - node->get_parameter(destination_name_ + ".database", database_); - node->get_parameter(destination_name_ + ".bucket", bucket_); - node->get_parameter(destination_name_ + ".org", org_); - node->get_parameter(destination_name_ + ".sequence_tag", sequence_tag_); - node->get_parameter(destination_name_ + ".http_user", http_user_); - node->get_parameter(destination_name_ + ".http_password", http_password_); - node->get_parameter(destination_name_ + ".http_token", http_token_); - node->get_parameter(destination_name_ + ".tag_keys", tag_keys_); - node->get_parameter(destination_name_ + ".auto_tags", auto_tags_); -} - -void FlbInfluxDB::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "influxdb", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "host", host_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "port", std::to_string(port_), NULL); - flb_output_set(ctx_, out_ffd_, "database", database_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "bucket", bucket_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "org", org_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "sequence_tag", sequence_tag_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "http_user", http_user_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "http_passwd", http_password_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "http_token", http_token_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "tag_keys", dc_util::join(tag_keys_, " ").c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "auto_tags", dc_util::boolToString(auto_tags_), NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output influxdb plugin"); - } -} - -FlbInfluxDB::~FlbInfluxDB() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbInfluxDB, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_kinesis_streams.cpp b/dc_destinations/plugins/flb_kinesis_streams.cpp deleted file mode 100644 index 3f2edbf29..000000000 --- a/dc_destinations/plugins/flb_kinesis_streams.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "dc_destinations/plugins/flb_kinesis_streams.hpp" - -namespace dc_destinations -{ - -FlbKinesisStreams::FlbKinesisStreams() : dc_destinations::FlbDestination() -{ -} - -void FlbKinesisStreams::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".region", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".stream", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".role_arn", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".time_key", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".time_key_format", - rclcpp::ParameterValue("%Y-%m-%dT%H:%M:%S")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".log_key", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".endpoint", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".auto_retry_requests", - rclcpp::ParameterValue("true")); - - node->get_parameter(destination_name_ + ".region", region_); - node->get_parameter(destination_name_ + ".stream", stream_); - node->get_parameter(destination_name_ + ".role_arn", role_arn_); - node->get_parameter(destination_name_ + ".time_key", time_key_); - node->get_parameter(destination_name_ + ".time_key_format", time_key_format_); - node->get_parameter(destination_name_ + ".log_key", log_key_); - node->get_parameter(destination_name_ + ".endpoint", endpoint_); - node->get_parameter(destination_name_ + ".auto_retry_requests", auto_retry_requests_); -} - -void FlbKinesisStreams::initFlbOutputPlugin() -{ - /* Enable output plugin 'stdout' (print records to the standard output) */ - out_ffd_ = flb_output(ctx_, "kinesis_streams", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "stream", stream_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "role_arn", role_arn_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "time_key", time_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "time_key_format", time_key_format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "log_key", log_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "endpoint", endpoint_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "auto_retry_requests", dc_util::boolToString(auto_retry_requests_), NULL); - - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output kinesis_streams plugin"); - } -} - -FlbKinesisStreams::~FlbKinesisStreams() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbKinesisStreams, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_minio.cpp b/dc_destinations/plugins/flb_minio.cpp deleted file mode 100644 index dbba4e0cf..000000000 --- a/dc_destinations/plugins/flb_minio.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "dc_destinations/plugins/flb_minio.hpp" - -namespace dc_destinations -{ - -FlbMinIO::FlbMinIO() : dc_destinations::FlbDestination() -{ -} - -void FlbMinIO::onConfigure() -{ - auto node = getNode(); - - std::string package_directory = ament_index_cpp::get_package_prefix("fluent_bit_plugins"); - plugin_path_default_ = package_directory + "/lib/out_minio.so"; - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".plugin_path", - rclcpp::ParameterValue(plugin_path_default_)); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".verbose_plugin", - rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".endpoint", - rclcpp::ParameterValue("127.0.0.1:9000")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".access_key_id", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".secret_access_key", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".use_ssl", rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".create_bucket", rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".bucket", rclcpp::ParameterValue("dc_bucket")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".upload_fields", - rclcpp::PARAMETER_STRING_ARRAY); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".src_fields", rclcpp::PARAMETER_STRING_ARRAY); - - node->get_parameter(destination_name_ + ".verbose_plugin", verbose_plugin_); - node->get_parameter(destination_name_ + ".plugin_path", plugin_path_); - node->get_parameter(destination_name_ + ".endpoint", endpoint_); - node->get_parameter(destination_name_ + ".access_key_id", access_key_id_); - node->get_parameter(destination_name_ + ".secret_access_key", secret_access_key_); - node->get_parameter(destination_name_ + ".use_ssl", use_ssl_); - node->get_parameter(destination_name_ + ".create_bucket", create_bucket_); - node->get_parameter(destination_name_ + ".bucket", bucket_); - node->get_parameter(destination_name_ + ".upload_fields", upload_fields_); - node->get_parameter(destination_name_ + ".src_fields", src_fields_); -} - -void FlbMinIO::initFlbOutputPlugin() -{ - RCLCPP_INFO(logger_, "Loading minio as GO proxy output %s", plugin_path_.c_str()); - if (flb_plugin_load_router(strdup(plugin_path_.c_str()), ctx_->config) != 0) - { - flb_error("[plugin] error loading GO proxy plugin: %s", plugin_path_.c_str()); - throw std::runtime_error("Cannot load plugin"); - } - - out_ffd_ = flb_output(ctx_, "minio", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - - flb_output_set(ctx_, out_ffd_, "verbose", dc_util::boolToString(verbose_plugin_), NULL); - flb_output_set(ctx_, out_ffd_, "retry_limit", "false", NULL); - flb_output_set(ctx_, out_ffd_, "endpoint", endpoint_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "access_key_id", access_key_id_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "secret_access_key", secret_access_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "use_ssl", dc_util::boolToString(use_ssl_), NULL); - flb_output_set(ctx_, out_ffd_, "create_bucket", dc_util::boolToString(create_bucket_), NULL); - flb_output_set(ctx_, out_ffd_, "bucket", bucket_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "src_fields", dc_util::to_space_separated_string(src_fields_).c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "upload_fields", dc_util::to_space_separated_string(upload_fields_).c_str(), NULL); -} - -FlbMinIO::~FlbMinIO() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbMinIO, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_null.cpp b/dc_destinations/plugins/flb_null.cpp deleted file mode 100644 index 7bfbf8c85..000000000 --- a/dc_destinations/plugins/flb_null.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "dc_destinations/plugins/flb_null.hpp" - -namespace dc_destinations -{ - -FlbNull::FlbNull() : dc_destinations::FlbDestination() -{ -} - -void FlbNull::onConfigure() -{ -} - -void FlbNull::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "null", NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output null plugin"); - } -} - -FlbNull::~FlbNull() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbNull, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_pgsql.cpp b/dc_destinations/plugins/flb_pgsql.cpp deleted file mode 100644 index 440100869..000000000 --- a/dc_destinations/plugins/flb_pgsql.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "dc_destinations/plugins/flb_pgsql.hpp" - -namespace dc_destinations -{ - -FlbPgSQL::FlbPgSQL() : dc_destinations::FlbDestination() -{ -} - -void FlbPgSQL::onConfigure() -{ - auto node = getNode(); - char username[MAX_USERID_LENGTH]; - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".host", rclcpp::ParameterValue("127.0.0.1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".port", rclcpp::ParameterValue(5432)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".user", - rclcpp::ParameterValue(cuserid(username))); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".password", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".database", - rclcpp::ParameterValue(cuserid(username))); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".table", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".timestamp_Key", - rclcpp::ParameterValue("date")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".async", rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".min_pool_size", rclcpp::ParameterValue("1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".max_pool_size", rclcpp::ParameterValue("4")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".cockroachdb", rclcpp::ParameterValue(false)); - node->get_parameter(destination_name_ + ".host", host_); - node->get_parameter(destination_name_ + ".port", port_); - node->get_parameter(destination_name_ + ".user", user_); - node->get_parameter(destination_name_ + ".password", password_); - node->get_parameter(destination_name_ + ".database", database_); - node->get_parameter(destination_name_ + ".table", table_); - node->get_parameter(destination_name_ + ".timestamp_key", timestamp_key_); - node->get_parameter(destination_name_ + ".async", async_); - node->get_parameter(destination_name_ + ".min_pool_size", min_pool_size_); - node->get_parameter(destination_name_ + ".max_pool_size", max_pool_size_); - node->get_parameter(destination_name_ + ".cockroachdb", cockroachdb_); -} - -void FlbPgSQL::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "pgsql", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "host", host_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "port", std::to_string(port_).c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "user", user_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "password", password_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "database", database_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "table", table_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "timestamp_key", timestamp_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "async", dc_util::boolToString(async_), NULL); - flb_output_set(ctx_, out_ffd_, "min_pool_size", min_pool_size_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "max_pool_size", max_pool_size_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "cockroachdb", dc_util::boolToString(cockroachdb_), NULL); - - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output pgsql plugin"); - } -} - -FlbPgSQL::~FlbPgSQL() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbPgSQL, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_s3.cpp b/dc_destinations/plugins/flb_s3.cpp deleted file mode 100644 index bc77e3c1b..000000000 --- a/dc_destinations/plugins/flb_s3.cpp +++ /dev/null @@ -1,128 +0,0 @@ -#include "dc_destinations/plugins/flb_s3.hpp" - -namespace dc_destinations -{ - -FlbS3::FlbS3() : dc_destinations::FlbDestination() -{ -} - -void FlbS3::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".region", rclcpp::ParameterValue("us-east-1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".bucket", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_key", - rclcpp::ParameterValue("date")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_format", - rclcpp::ParameterValue("iso8601")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".total_file_size", - rclcpp::ParameterValue("100M")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".upload_chunk_size", - rclcpp::ParameterValue("50M")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".upload_timeout", - rclcpp::ParameterValue("10m")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".store_dir", - rclcpp::ParameterValue("/tmp/fluent-bit/s3")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".store_dir_limit_size", - rclcpp::ParameterValue(0)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3_key_format", - rclcpp::ParameterValue("/fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".static_file_path", - rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".s3_key_format_tag_delimiters", - rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".use_put_object", - rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".role_arn", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".endpoint", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".sts_endpoint", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".canned_acl", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".compression", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".content_type", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".send_content_md5", - rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".auto_retry_requests", - rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".log_key", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".preserve_data_ordering", - rclcpp::ParameterValue(true)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".storage_class", rclcpp::ParameterValue("")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".retry_limit", rclcpp::ParameterValue(1)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".external_id", rclcpp::ParameterValue("")); - node->get_parameter(destination_name_ + ".region", region_); - node->get_parameter(destination_name_ + ".bucket", bucket_); - node->get_parameter(destination_name_ + ".json_date_key", json_date_key_); - node->get_parameter(destination_name_ + ".json_date_format", json_date_format_); - node->get_parameter(destination_name_ + ".total_file_size", total_file_size_); - node->get_parameter(destination_name_ + ".upload_chunk_size", upload_chunk_size_); - node->get_parameter(destination_name_ + ".upload_timeout", upload_timeout_); - node->get_parameter(destination_name_ + ".store_dir", store_dir_); - node->get_parameter(destination_name_ + ".store_dir_limit_size", store_dir_limit_size_); - node->get_parameter(destination_name_ + ".s3_key_format", s3_key_format_); - node->get_parameter(destination_name_ + ".s3_key_format_tag_delimiters", s3_key_format_tag_delimiters_); - node->get_parameter(destination_name_ + ".static_file_path", static_file_path_); - node->get_parameter(destination_name_ + ".use_put_object", use_put_object_); - node->get_parameter(destination_name_ + ".role_arn", role_arn_); - node->get_parameter(destination_name_ + ".endpoint", endpoint_); - node->get_parameter(destination_name_ + ".sts_endpoint", sts_endpoint_); - node->get_parameter(destination_name_ + ".canned_acl", canned_acl_); - node->get_parameter(destination_name_ + ".compression", compression_); - node->get_parameter(destination_name_ + ".content_type", content_type_); - node->get_parameter(destination_name_ + ".send_content_md5", send_content_md5_); - node->get_parameter(destination_name_ + ".auto_retry_requests", auto_retry_requests_); - node->get_parameter(destination_name_ + ".log_key", log_key_); - node->get_parameter(destination_name_ + ".preserve_data_ordering", preserve_data_ordering_); - node->get_parameter(destination_name_ + ".storage_class", storage_class_); - node->get_parameter(destination_name_ + ".retry_limit", retry_limit_); - node->get_parameter(destination_name_ + ".role_arn", role_arn_); - node->get_parameter(destination_name_ + ".external_id", external_id_); -} - -void FlbS3::initFlbOutputPlugin() -{ - /* Enable output plugin 'stdout' (print records to the standard output) */ - out_ffd_ = flb_output(ctx_, "s3", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - - flb_output_set(ctx_, out_ffd_, "region", region_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "bucket", bucket_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_key", json_date_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_format", json_date_format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "total_file_size", total_file_size_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "upload_chunk_size", upload_chunk_size_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "upload_timeout", upload_timeout_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "store_dir", store_dir_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "store_dir_limit_size", std::to_string(store_dir_limit_size_).c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "s3_key_format", s3_key_format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "s3_key_format_tag_delimiters", s3_key_format_tag_delimiters_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "static_file_path", dc_util::boolToString(static_file_path_), NULL); - flb_output_set(ctx_, out_ffd_, "use_put_object", dc_util::boolToString(use_put_object_), NULL); - flb_output_set(ctx_, out_ffd_, "role_arn", role_arn_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "endpoint", endpoint_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "sts_endpoint", sts_endpoint_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "canned_acl", canned_acl_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "compression", compression_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "content_type", content_type_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "send_content_md5", dc_util::boolToString(send_content_md5_), NULL); - flb_output_set(ctx_, out_ffd_, "auto_retry_requests", dc_util::boolToString(auto_retry_requests_), NULL); - flb_output_set(ctx_, out_ffd_, "log_key", log_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "preserve_data_ordering", dc_util::boolToString(preserve_data_ordering_), NULL); - flb_output_set(ctx_, out_ffd_, "storage_class", storage_class_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "retry_limit", retry_limit_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "external_id", external_id_.c_str(), NULL); - - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output s3 plugin"); - } -} - -FlbS3::~FlbS3() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbS3, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_slack.cpp b/dc_destinations/plugins/flb_slack.cpp deleted file mode 100644 index 79bdae004..000000000 --- a/dc_destinations/plugins/flb_slack.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "dc_destinations/plugins/flb_slack.hpp" - -namespace dc_destinations -{ - -FlbSlack::FlbSlack() : dc_destinations::FlbDestination() -{ -} - -void FlbSlack::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".webhook", rclcpp::PARAMETER_STRING); - node->get_parameter(destination_name_ + ".webhook", webhook_); -} - -void FlbSlack::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "slack", NULL); - flb_output_set(ctx_, out_ffd_, "webhook", webhook_.c_str(), NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output slack plugin"); - } -} - -FlbSlack::~FlbSlack() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbSlack, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_stdout.cpp b/dc_destinations/plugins/flb_stdout.cpp deleted file mode 100644 index 6e526686f..000000000 --- a/dc_destinations/plugins/flb_stdout.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "dc_destinations/plugins/flb_stdout.hpp" - -namespace dc_destinations -{ - -FlbStdout::FlbStdout() : dc_destinations::FlbDestination() -{ -} - -void FlbStdout::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".format", rclcpp::ParameterValue("json")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_key", - rclcpp::ParameterValue("date")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_format", - rclcpp::ParameterValue("double")); - node->get_parameter(destination_name_ + ".format", format_); - node->get_parameter(destination_name_ + ".json_date_key", json_date_key_); - node->get_parameter(destination_name_ + ".json_date_format", json_date_format_); -} - -void FlbStdout::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "stdout", NULL); - flb_output_set(ctx_, out_ffd_, "match", destination_name_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "format", format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_key", json_date_key_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_format", json_date_format_.c_str(), NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output stdout plugin"); - } -} - -FlbStdout::~FlbStdout() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbStdout, dc_core::Destination) diff --git a/dc_destinations/plugins/flb_tcp.cpp b/dc_destinations/plugins/flb_tcp.cpp deleted file mode 100644 index 762c74493..000000000 --- a/dc_destinations/plugins/flb_tcp.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "dc_destinations/plugins/flb_tcp.hpp" - -namespace dc_destinations -{ - -FlbTCP::FlbTCP() : dc_destinations::FlbDestination() -{ -} - -void FlbTCP::onConfigure() -{ - auto node = getNode(); - - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".host", rclcpp::ParameterValue("127.0.0.1")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".port", rclcpp::ParameterValue(5432)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".format", rclcpp::ParameterValue("json")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_key", - rclcpp::ParameterValue("date")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".json_date_format", - rclcpp::ParameterValue("double")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".workers", rclcpp::ParameterValue(2)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.active", rclcpp::ParameterValue("off")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.verify", rclcpp::ParameterValue("on")); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.debug", rclcpp::ParameterValue(1)); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.ca_file", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.crt_file", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.key_file", rclcpp::PARAMETER_STRING); - nav2_util::declare_parameter_if_not_declared(node, destination_name_ + ".tls.key_passwd", rclcpp::PARAMETER_STRING); - node->get_parameter(destination_name_ + ".host", host_); - node->get_parameter(destination_name_ + ".port", port_); - node->get_parameter(destination_name_ + ".format", format_); - node->get_parameter(destination_name_ + ".json_date_key", json_date_key_); - node->get_parameter(destination_name_ + ".json_date_format", json_date_format_); - node->get_parameter(destination_name_ + ".workers", workers_); - node->get_parameter(destination_name_ + ".tls.active", tls_active_); - node->get_parameter(destination_name_ + ".tls.verify", tls_verify_); - node->get_parameter(destination_name_ + ".tls.debug", tls_debug_); - node->get_parameter(destination_name_ + ".tls.ca_file", tls_ca_file_); - node->get_parameter(destination_name_ + ".tls.crt_file", tls_crt_file_); - node->get_parameter(destination_name_ + ".tls.key_file", tls_key_file_); - node->get_parameter(destination_name_ + ".tls.key_passwd", tls_key_passwd_); -} - -void FlbTCP::initFlbOutputPlugin() -{ - out_ffd_ = flb_output(ctx_, "tcp", NULL); - flb_output_set(ctx_, out_ffd_, "host", host_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "port", std::to_string(port_), NULL); - flb_output_set(ctx_, out_ffd_, "format", format_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "workers", std::to_string(workers_), NULL); - flb_output_set(ctx_, out_ffd_, "json_date_key", json_date_key_, NULL); - flb_output_set(ctx_, out_ffd_, "json_date_format", json_date_format_, NULL); - flb_output_set(ctx_, out_ffd_, "tls.active", dc_util::boolToString(tls_active_), NULL); - flb_output_set(ctx_, out_ffd_, "tls.verify", dc_util::boolToString(tls_verify_), NULL); - flb_output_set(ctx_, out_ffd_, "tls.debug", std::to_string(tls_debug_), NULL); - flb_output_set(ctx_, out_ffd_, "tls.ca_file", tls_ca_file_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "tls.crt_file", tls_crt_file_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "tls.key_file", tls_key_file_.c_str(), NULL); - flb_output_set(ctx_, out_ffd_, "tls.key_passwd", tls_key_passwd_.c_str(), NULL); - if (out_ffd_ == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit output tcp plugin"); - } -} - -FlbTCP::~FlbTCP() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::FlbTCP, dc_core::Destination) diff --git a/dc_destinations/plugins/rcl.cpp b/dc_destinations/plugins/rcl.cpp deleted file mode 100644 index 20a46ec25..000000000 --- a/dc_destinations/plugins/rcl.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "dc_destinations/plugins/rcl.hpp" - -#include -#include - -namespace dc_destinations -{ - -Rcl::Rcl() : dc_destinations::Destination() -{ -} - -void Rcl::sendData(dc_interfaces::msg::StringStamped::SharedPtr msg) -{ - json data = json::parse(msg->data); - data["date"] = - std::stod(std::to_string(msg->header.stamp.sec) + std::string(".") + std::to_string(msg->header.stamp.nanosec)); - data.erase("tags"); - - RCLCPP_INFO(logger_, "%s", data.dump().c_str()); -} - -Rcl::~Rcl() = default; - -} // namespace dc_destinations - -#include "pluginlib/class_list_macros.hpp" -PLUGINLIB_EXPORT_CLASS(dc_destinations::Rcl, dc_core::Destination) diff --git a/dc_destinations/src/destination_server.cpp b/dc_destinations/src/destination_server.cpp deleted file mode 100644 index c0f51dd8c..000000000 --- a/dc_destinations/src/destination_server.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "dc_destinations/destination_server.hpp" - -#include -#include -#include -#include - -namespace destination_server -{ - -DestinationServer::DestinationServer(const rclcpp::NodeOptions& options) - : nav2_util::LifecycleNode("destination_server", "", options), plugin_loader_("dc_core", "dc_core::Destination") -{ -} - -DestinationServer::~DestinationServer() -{ - destinations_.clear(); -} - -nav2_util::CallbackReturn DestinationServer::on_configure(const rclcpp_lifecycle::State& /*state*/) -{ - declare_parameter("destination_plugins", default_ids_); - get_parameter("destination_plugins", destination_ids_); - - // Fluent Bit - // https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file#config_section - nav2_util::declare_parameter_if_not_declared(this, "flb.flush", rclcpp::ParameterValue(1)); - nav2_util::declare_parameter_if_not_declared(this, "flb.grace", rclcpp::ParameterValue(1)); - nav2_util::declare_parameter_if_not_declared(this, "flb.log_level", rclcpp::ParameterValue("info")); - nav2_util::declare_parameter_if_not_declared(this, "flb.storage_path", - rclcpp::ParameterValue("/var/log/flb-storage/")); - nav2_util::declare_parameter_if_not_declared(this, "flb.storage_sync", rclcpp::ParameterValue("normal")); - nav2_util::declare_parameter_if_not_declared(this, "flb.storage_checksum", rclcpp::ParameterValue("off")); - nav2_util::declare_parameter_if_not_declared(this, "flb.storage_backlog_mem_limit", rclcpp::ParameterValue("5M")); - nav2_util::declare_parameter_if_not_declared(this, "flb.scheduler_cap", rclcpp::ParameterValue(2000)); - nav2_util::declare_parameter_if_not_declared(this, "flb.scheduler_base", rclcpp::ParameterValue(5)); - nav2_util::declare_parameter_if_not_declared(this, "flb.http_server", rclcpp::ParameterValue(false)); - nav2_util::declare_parameter_if_not_declared(this, "flb.http_listen", rclcpp::ParameterValue("0.0.0.0")); - nav2_util::declare_parameter_if_not_declared(this, "flb.http_port", rclcpp::ParameterValue(2020)); - nav2_util::declare_parameter_if_not_declared(this, "flb.in_storage_type", rclcpp::ParameterValue("filesystem")); - nav2_util::declare_parameter_if_not_declared(this, "flb.in_storage_pause_on_chunks_overlimit", - rclcpp::ParameterValue("off")); - - std::string package_directory = ament_index_cpp::get_package_prefix("fluent_bit_plugins"); - ros2_plugin_path_default_ = package_directory + "/lib/flb-in_ros2.so"; - nav2_util::declare_parameter_if_not_declared(this, "ros2_plugin_path", - rclcpp::ParameterValue(ros2_plugin_path_default_)); - nav2_util::declare_parameter_if_not_declared(this, "ros2_plugin_spin_time_ms", rclcpp::ParameterValue(100)); - - flb_flush_ = this->get_parameter("flb.flush").as_int(); - flb_grace_ = this->get_parameter("flb.grace").as_int(); - flb_log_level_ = this->get_parameter("flb.log_level").as_string(); - flb_storage_path_ = this->get_parameter("flb.storage_path").as_string(); - flb_storage_sync_ = this->get_parameter("flb.storage_sync").as_string(); - flb_storage_checksum_ = this->get_parameter("flb.storage_checksum").as_string(); - flb_storage_backlog_mem_limit_ = this->get_parameter("flb.storage_backlog_mem_limit").as_string(); - flb_scheduler_cap_ = this->get_parameter("flb.scheduler_cap").as_int(); - flb_scheduler_base_ = this->get_parameter("flb.scheduler_base").as_int(); - flb_http_server_ = this->get_parameter("flb.http_server").as_bool(); - flb_http_listen_ = this->get_parameter("flb.http_listen").as_string(); - flb_http_port_ = this->get_parameter("flb.http_port").as_int(); - flb_in_storage_type_ = this->get_parameter("flb.in_storage_type").as_string(); - flb_in_storage_pause_on_chunks_overlimit_ = - this->get_parameter("flb.in_storage_pause_on_chunks_overlimit").as_string(); - ros2_plugin_path_ = this->get_parameter("ros2_plugin_path").as_string(); - ros2_plugin_spin_time_ms_ = this->get_parameter("ros2_plugin_spin_time_ms").as_int(); - - destination_types_.resize(destination_ids_.size()); - destination_inputs_.resize(destination_ids_.size()); - destination_debug_.resize(destination_ids_.size()); - destination_time_format_.resize(destination_ids_.size()); - destination_time_key_.resize(destination_ids_.size()); - - initFlb(); - - if (!loadDestinationPlugins()) - { - return nav2_util::CallbackReturn::FAILURE; - } - - initFlbInputPlugin(); - startFlbEngine(); - - return nav2_util::CallbackReturn::SUCCESS; -} - -bool DestinationServer::loadDestinationPlugins() -{ - auto node = shared_from_this(); - - for (size_t i = 0; i != destination_ids_.size(); i++) - { - // Mandatory parameters - destination_types_[i] = dc_util::get_str_type_param(node, destination_ids_[i], "plugin"); - destination_inputs_[i] = dc_util::get_str_array_type_param(node, destination_ids_[i], "inputs"); - destination_debug_[i] = dc_util::get_bool_type_param(node, destination_ids_[i], "debug", false); - destination_time_format_[i] = dc_util::get_str_type_param(node, destination_ids_[i], "time_format", "double"); - destination_time_key_[i] = dc_util::get_str_type_param(node, destination_ids_[i], "time_key", "date"); - - try - { - RCLCPP_INFO_STREAM(get_logger(), "Creating destination plugin " - << destination_ids_[i].c_str() << ": Type " << destination_types_[i].c_str() - << ", Debug: " << (int)destination_debug_[i] - << ", Time format: " << destination_time_format_[i] - << ". Time key: " << destination_time_key_[i]); - - destinations_.push_back(plugin_loader_.createUniqueInstance(destination_types_[i])); - destinations_.back()->configure(node, destination_ids_[i], destination_inputs_[i], ctx_, destination_debug_[i], - flb_in_storage_type_, destination_time_format_[i], destination_time_key_[i]); - } - catch (const pluginlib::PluginlibException& ex) - { - RCLCPP_FATAL(get_logger(), - "Failed to create destination %s of type %s, debug %d" - " Exception: %s", - destination_ids_[i].c_str(), destination_types_[i].c_str(), (int)destination_debug_[i], ex.what()); - return false; - } - } - - return true; -} - -void DestinationServer::initFlb() -{ - /* Create library context */ - mk_core_init(); - ctx_ = flb_create(); - - /* Enable metrics if option enabled */ - std::string http_server = ""; - if (flb_http_server_) - { - http_server = "on"; - } - else - { - http_server = "off"; - } - - flb_service_set(ctx_, "flush", std::to_string(flb_flush_).c_str(), "grace", std::to_string(flb_grace_).c_str(), - "log_Level", flb_log_level_.c_str(), "storage.path", flb_storage_path_.c_str(), "storage.sync", - flb_storage_sync_.c_str(), "storage.checksum", flb_storage_checksum_.c_str(), - "storage.backlog.mem_limit", flb_storage_backlog_mem_limit_.c_str(), "scheduler.cap", - std::to_string(flb_scheduler_cap_).c_str(), "scheduler.base", - std::to_string(flb_scheduler_base_).c_str(), "http_server", http_server.c_str(), "http_listen", - flb_http_listen_.c_str(), "http_port", std::to_string(flb_http_port_).c_str(), NULL); - - if (!ctx_) - { - throw std::runtime_error("Cannot create Fluent Bit library context"); - } - - RCLCPP_INFO(get_logger(), "Fluent Bit service initialized"); -} - -void DestinationServer::startFlbEngine() -{ - /* Start the engine */ - RCLCPP_INFO(get_logger(), "Starting Flb engine..."); - int ret = flb_start(ctx_); - if (ret == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot start Fluent Bit engine"); - } - RCLCPP_INFO(get_logger(), "Started Flb engine"); -} - -void DestinationServer::initFlbInputPlugin() -{ - int in_ffd; - int ret = 0; - - /* Enable the input plugin for manual data ingestion */ - RCLCPP_INFO(get_logger(), "Loading input ros2 shared library %s...", ros2_plugin_path_.c_str()); - if (flb_plugin_load_router(strdup(ros2_plugin_path_.c_str()), ctx_->config) != 0) - { - flb_error("[plugin] error loading c plugin: %s", ros2_plugin_path_.c_str()); - throw std::runtime_error("Cannot load plugin"); - } - - RCLCPP_INFO(get_logger(), "Loaded input ros2 shared library %s", ros2_plugin_path_.c_str()); - - in_ffd = flb_input(ctx_, "ros2", NULL); - if (in_ffd == -1) - { - flb_destroy(ctx_); - throw std::runtime_error("Cannot initialize Fluent Bit input ros2 plugin"); - } - - ros_topics_ = dc_util::remove_duplicates(dc_util::flatten(destination_inputs_)); - - ret += flb_input_set(ctx_, in_ffd, "tag", "ros2", NULL); - - ret += flb_input_set(ctx_, in_ffd, "topics", dc_util::to_space_separated_string(ros_topics_).c_str(), NULL); - ret += flb_input_set(ctx_, in_ffd, "spin_time", std::to_string(ros2_plugin_spin_time_ms_).c_str(), NULL); - ret += flb_input_set(ctx_, in_ffd, "storage.type", flb_in_storage_type_.c_str(), NULL); - ret += flb_input_set(ctx_, in_ffd, "storage.pause_on_chunks_overlimit", - flb_in_storage_pause_on_chunks_overlimit_.c_str(), NULL); - - if (ret != 0) - { - throw std::runtime_error(std::string("Cannot initialize parameters Fluent Bit input ros2 plugin. topics: \"") + - dc_util::to_space_separated_string(ros_topics_) + "\", storage.type: \"" + - flb_in_storage_type_ + "\", storage.pause_on_chunks_overlimit: \"" + - flb_in_storage_pause_on_chunks_overlimit_ + "\""); - } - - RCLCPP_INFO(get_logger(), "Flb ros2 plugin initialized. ret=%d", ret); -} - -nav2_util::CallbackReturn DestinationServer::on_activate(const rclcpp_lifecycle::State& /*previous_state*/) -{ - std::vector>::iterator iter; - for (iter = destinations_.begin(); iter != destinations_.end(); ++iter) - { - (*iter)->activate(); - } - - // create bond connection - createBond(); - - return nav2_util::CallbackReturn::SUCCESS; -} - -nav2_util::CallbackReturn DestinationServer::on_deactivate(const rclcpp_lifecycle::State& /*state*/) -{ - RCLCPP_INFO(get_logger(), "Deactivating"); - - std::vector>::iterator iter; - for (iter = destinations_.begin(); iter != destinations_.end(); ++iter) - { - (*iter)->deactivate(); - } - - // destroy bond connection - destroyBond(); - - return nav2_util::CallbackReturn::SUCCESS; -} - -nav2_util::CallbackReturn DestinationServer::on_cleanup(const rclcpp_lifecycle::State& /*state*/) -{ - RCLCPP_INFO(get_logger(), "Cleaning up"); - - return nav2_util::CallbackReturn::SUCCESS; -} - -nav2_util::CallbackReturn DestinationServer::on_shutdown(const rclcpp_lifecycle::State& /*previous_state*/) -{ - RCLCPP_INFO(get_logger(), "Shutting down"); - return nav2_util::CallbackReturn::SUCCESS; -} - -} // end namespace destination_server - -#include "rclcpp_components/register_node_macro.hpp" - -// Register the component with class_loader. -// This acts as a sort of entry point, allowing the component to be discoverable when its library -// is being loaded into a running process. -RCLCPP_COMPONENTS_REGISTER_NODE(destination_server::DestinationServer) diff --git a/dc_destinations/src/main.cpp b/dc_destinations/src/main.cpp deleted file mode 100644 index c971f1045..000000000 --- a/dc_destinations/src/main.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include - -#include "dc_destinations/destination_server.hpp" -#include "rclcpp/rclcpp.hpp" - -int main(int argc, char** argv) -{ - rclcpp::init(argc, argv); - rclcpp::executors::MultiThreadedExecutor executor; - auto destination_node = std::make_shared(); - - executor.add_node(destination_node->get_node_base_interface()); - executor.spin(); - rclcpp::shutdown(); - - return 0; -} diff --git a/dc_measurements/include/dc_measurements/measurement.hpp b/dc_measurements/include/dc_measurements/measurement.hpp index 46c9c40e3..02e2f6b06 100644 --- a/dc_measurements/include/dc_measurements/measurement.hpp +++ b/dc_measurements/include/dc_measurements/measurement.hpp @@ -231,7 +231,7 @@ class Measurement : public dc_core::Measurement void addTags(dc_interfaces::msg::StringStamped& msg) { // Only put the tags in if the conditions are ok. This way, we still publish the data - // and can check in conditions but with no tags, this is not received by the destination_server + // and can check in conditions but with no tags, this is not received by the Bridge if (!tags_.empty()) { try @@ -267,7 +267,7 @@ class Measurement : public dc_core::Measurement void addMeasurementName(dc_interfaces::msg::StringStamped& msg) { // Only put the tags in if the conditions are ok. This way, we still publish the data - // and can check in conditions but with no tags, this is not received by the destination_server + // and can check in conditions but with no tags, this is not received by the Bridge if (include_measurement_name_) { try @@ -287,7 +287,7 @@ class Measurement : public dc_core::Measurement void addMeasurementPluginName(dc_interfaces::msg::StringStamped& msg) { // Only put the tags in if the conditions are ok. This way, we still publish the data - // and can check in conditions but with no tags, this is not received by the destination_server + // and can check in conditions but with no tags, this is not received by the Bridge if (include_measurement_plugin_) { try diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 1e8cd0416..2a6efbbd0 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -49,19 +49,6 @@ - [Data validation](./dc/data_validation.md) - [Groups](./dc/groups.md) - [Destinations](./dc/destinations.md) - - [Fluent Bit AWS Kinesis Data Streams](./dc/destinations/flb_kinesis_data_streams.md) - - [Fluent Bit AWS S3](./dc/destinations/flb_s3.md) - - [Fluent Bit File](./dc/destinations/flb_file.md) - - [Fluent Bit Files metrics](./dc/destinations/flb_files_metrics.md) - - [Fluent Bit HTTP](./dc/destinations/flb_http.md) - - [Fluent Bit InfluxDB](./dc/destinations/flb_influxdb.md) - - [Fluent Bit MinIO](./dc/destinations/flb_minio.md) - - [Fluent Bit NULL](./dc/destinations/flb_null.md) - - [Fluent Bit PostgreSQL](./dc/destinations/flb_pgsql.md) - - [Fluent Bit Slack](./dc/destinations/flb_slack.md) - - [Fluent Bit Stdout](./dc/destinations/flb_stdout.md) - - [Fluent Bit TCP](./dc/destinations/flb_tcp.md) - - [RCL](./dc/destinations/rcl.md) - [Configuration examples](./dc/configuration_examples.md) - [Infrastructure setup](./dc/infrastructure_setup.md) - [InfluxDB](./dc/infrastructure_setup/influxdb.md) diff --git a/doc/src/dc/concepts.md b/doc/src/dc/concepts.md index 5114e4260..2e03040f6 100644 --- a/doc/src/dc/concepts.md +++ b/doc/src/dc/concepts.md @@ -6,13 +6,16 @@ There are a few key concepts that are really important to understand how DC oper ROS 2 is the core middleware used for DC. If you are unfamilar with this, please visit [the ROS 2 documentation](https://docs.ros.org/en/rolling/) before continuing. -## Fluent Bit +## Shipper (Vector) -Fluent Bit is used in the backend for most plugins in DC. If you are unfamilar with this, please visit [the Fluent Bit documentation](https://docs.fluentbit.io/manual/) - -The DC destination ROS node starts it using the fluent bit C api and thus DC directly gets all benefits from it. - -Fluent Bit configuration has been wrapped in this node and all its configuration parameters can be passed from the YAML configuration file. +DC's data plane is an external **Shipper** process, [Vector](https://vector.dev/), +fed over the Fluent Forward protocol by the **Bridge** (`dc_bridge`). The Bridge +renders Vector's configuration from plain ROS parameters — see +[Destinations](./destinations.md) — spawns and supervises the Vector process, and +forwards every Record it receives on its configured input topics. DC gets Vector's +disk buffering, retries, and native sinks (PostgreSQL, S3-compatible storage, and +many more) without embedding or forking the Shipper itself (ADR-0001, +`docs/adr/`). ## Lifecycle Nodes and Bond *(Source: [Nav2 documentation](https://navigation.ros.org/concepts/index.html))* @@ -44,23 +47,24 @@ Measurements are a single data unit presented in JSON format, that can contain d Every incoming piece of data that belongs to a log or a metric that is retrieved by DC is considered an Event or Record. -Internally, when using Fluent Bit based plugins, it will contain 2 components: its timestamp and its message. For us, in DC, it will always be a JSON string sent in a ROS message of StringStamped type. This ROS message contains: +Internally, it will always be a JSON string sent in a ROS message of StringStamped type. This ROS message contains: 1. **header**: ROS timestamp as std_msgs/Header 2. **data**: JSON message as string 3. **group_key**: a string used as a key for the new message when grouping multiple messages together ## Tag(s) -Every measurement requires to have at least a tag configured (via the `tags` parameter) so it is sent to its destination(s). This tag corresponds to the name of the plugin you defined in the same configuration. It is then used in a later stage by the Router to decide which Filter or Output phase it must go through. +Every measurement requires to have at least a tag configured (via the `tags` parameter) so it is sent to its destination(s). This tag corresponds to the name of a Destination declared on the Bridge (`dc_bridge`) in the same configuration. Example: ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: # Custom name for the plugin - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: # Custom name for the Destination + type: console + receives: records inputs: ["/dc/measurement/string_stamped"] measurement_server: ros__parameters: @@ -68,21 +72,17 @@ measurement_server: uptime: plugin: "dc_measurements/Uptime" topic_output: "/dc/measurement/uptime" - tags: ["flb_stdout"] # Match the plugin set in the destination_server + tags: ["console"] # Match the Destination name set on the Bridge ``` -To manage the tags in DC, we pass the tags as parameter to each measurement and automatically use 2 Fluent Bit filters to assign the ROS message to a certain Fluent Bit output: - -1. rewrite_tag filter: Modify the message tag to the destination(s) configured -2. lua filter: Take the flags received as a string containing a list and set the tags internally in Fluent Bit. - -You can find th code in flb_destination.hpp - -## Match -Fluent Bit allows to deliver your collected and processed Events to one or multiple destinations, this is done through a routing phase. A Match represent a simple rule to select Events where its tags matches a defined rule. +See [Destinations](./destinations.md) for the full config renderer contract, including +`inputs`/`tags` matching and the `dc.` routing convention. ## Destinations -A destination is where the data will be sent: AWS S3, stdout, AWS Kinesis. It has the possibility to use [outputs from fluentbit](https://docs.fluentbit.io/manual/pipeline/outputs). +A destination is where the data will be sent: PostgreSQL, S3-compatible storage, a +file, or the console are blessed (rendered from plain ROS parameters); any other +Vector sink is reachable through the passthrough. See [Destinations](./destinations.md) +for the full contract. ## Conditions A condition enables or disables one or multiple measurements to be published and thus collected. We could for example enable collecting camera images only when a robot is stopped. @@ -110,20 +110,17 @@ Each measurement has its own JSON schema, which can be overwritten in a custom p ## Buffering and data persistence -Fluent Bit has its own buffering management, explained in [its documentation](https://docs.fluentbit.io/manual/concepts/buffering). Data can be stored in Memory or/and filesystem. - -By default, DC uses the memory buffering for a small amount of data (5M) and then uses filesystem buffering. - -The configuration is printed when starting the destination server: - -```bash -[destination_server-2] [2023/02/24 10:36:15] [ info] [storage] ver=1.3.0, type=memory+filesystem, sync=full, checksum=off, max_chunks_up=128 -[destination_server-2] [2023/02/24 10:36:15] [ info] [input:ros2:ros2.0] storage_strategy='filesystem' (memory + filesystem) -[destination_server-2] [2023/02/24 10:36:15] [ info] [input:storage_backlog:storage_backlog.1] queue memory limit: 976.6K -``` - -This buffering ensures data will persist across reboots +Vector, the Shipper, manages its own disk buffer (`shipper.data_dir` on the Bridge's +parameters) with a documented minimum size (`shipper.buffer_max_bytes`). This buffer +persists across reboots: Records accepted by the Bridge but not yet delivered to a +Destination survive an outage of that Destination, a robot reboot, or a Bridge +restart, and are delivered — with end-to-end acknowledgements — once the Destination +is reachable again. ## Scheduling and Retries -DC inherits from Fluent Bit features of scheduling and retries. It can be configured in the destination node. More about it can be read on the [Fluent Bit documentation](https://docs.fluentbit.io/manual/administration/scheduling-and-retries) +Vector retries delivery to a Destination on failure with its own backoff, independent +of DC code; see [its documentation](https://vector.dev/docs/) for details. The Bridge +itself is supervised by DC (launch respawn) and, in turn, supervises its own Vector +child process — including a Linux parent-death signal so Vector can never outlive the +Bridge across a crash or SIGKILL. diff --git a/doc/src/dc/configuration_examples.md b/doc/src/dc/configuration_examples.md index b67f74e45..3c66f2db8 100644 --- a/doc/src/dc/configuration_examples.md +++ b/doc/src/dc/configuration_examples.md @@ -15,14 +15,15 @@ ros2 launch dc_bringup params_file:="my_file.yaml" ``` ## Running the examples -### Example 1: Uptime to Fluent Bit Stdout every second and flush data every second +### Example 1: Uptime to the console every second ```yaml -destination_server: # Destination node configuration +dc_bridge: # Bridge (Shipper) node configuration ros__parameters: - destination_plugins: ["flb_stdout"] # List of destination plugins names to enable - flb_stdout: # Plugin name, you choose - plugin: "dc_destinations/FlbStdout" # Plugin class name, fixed + destinations: ["console"] # List of Destination names to enable + console: # Destination name, you choose + type: console # Blessed Destination type, fixed + receives: records inputs: ["/dc/measurement/uptime"] # Same as topic_output in the uptime measurement in measurement_server measurement_server: # Measurement node configuration @@ -31,20 +32,21 @@ measurement_server: # Measurement node configuration uptime: # Plugin name, you choose plugin: "dc_measurements/Uptime" # Plugin class name, fixed topic_output: "/dc/measurement/uptime" # Topic where data will be published - tags: ["flb_stdout"] # Fluent Bit will match this in the destination server + tags: ["console"] # The Bridge will match this against a Destination name ``` -### Example 2: Uptime to Fluent Bit Stdout every second and flush data every 3 seconds +### Example 2: Uptime to the console with ISO 8601 timestamps ```yaml -destination_server: +dc_bridge: ros__parameters: - flb: - flush: 3 # Interval to flush output (seconds) - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: + type: console + receives: records inputs: ["/dc/measurement/uptime"] + time_key: "date" # Field the normalized timestamp is written to + time_format: "iso8601" # "double" (Unix epoch seconds) or "iso8601" measurement_server: ros__parameters: @@ -52,17 +54,18 @@ measurement_server: uptime: plugin: "dc_measurements/Uptime" topic_output: "/dc/measurement/uptime" - tags: ["flb_stdout"] + tags: ["console"] ``` -### Example 3: Uptime to Fluent Bit Stdout only at start and 3 times +### Example 3: Uptime to the console only at start and 3 times ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: + type: console + receives: records inputs: ["/dc/measurement/uptime"] measurement_server: @@ -71,18 +74,19 @@ measurement_server: uptime: plugin: "dc_measurements/Uptime" topic_output: "/dc/measurement/uptime" - tags: ["flb_stdout"] + tags: ["console"] init_max_measurements: 3 # Maximum records to collect ``` -### Example 4: CPU and Memory to Fluent Bit Stdout every 5 seconds forever +### Example 4: CPU and Memory to the console every 5 seconds forever ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: + type: console + receives: records inputs: ["/dc/measurement/cpu", "/dc/measurement/memory"] measurement_server: @@ -92,23 +96,24 @@ measurement_server: plugin: "dc_measurements/Memory" topic_output: "/dc/measurement/memory" polling_interval: 5000 # Interval to which data is collected in milliseconds - tags: ["flb_stdout"] + tags: ["console"] cpu: plugin: "dc_measurements/Cpu" topic_output: "/dc/measurement/cpu" polling_interval: 5000 # Interval to which data is collected in milliseconds - tags: ["flb_stdout"] + tags: ["console"] ``` -### Example 5: CPU and Memory as a group to Fluent Bit Stdout every 5 seconds forever +### Example 5: CPU and Memory as a group to the console every 5 seconds forever ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" - inputs: ["/dc/group/memory_cpu"] # Group to create + destinations: ["console"] + console: + type: console + receives: records + inputs: ["/dc/group/cpu_memory"] # Group to create group_server: # Group server configuration ros__parameters: @@ -118,7 +123,7 @@ group_server: # Group server configuration output: "/dc/group/cpu_memory" # Topic where result will be published sync_delay: 5.0 # How long to queue up messages before passing them through. group_key: "cpu_memory" - tags: ["flb_stdout"] + tags: ["console"] measurement_server: ros__parameters: @@ -127,22 +132,23 @@ measurement_server: plugin: "dc_measurements/Memory" topic_output: "/dc/measurement/memory" polling_interval: 5000 - tags: ["flb_stdout"] + tags: ["console"] cpu: plugin: "dc_measurements/Cpu" topic_output: "/dc/measurement/cpu" polling_interval: 5000 - tags: ["flb_stdout"] + tags: ["console"] ``` -### Example 6: Custom ROS message to Stdout every 2 seconds forever +### Example 6: Custom ROS message to the console every 2 seconds forever ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: + type: console + receives: records inputs: ["/dc/measurement/string_stamped"] measurement_server: @@ -152,7 +158,7 @@ measurement_server: plugin: "dc_measurements/StringStamped" # Plugin that allow to publish from your nodes topic_output: "/dc/measurement/my_string_stamped" # Topic where data is republished with tags topic: "/hello-world" # Input topic where you are publishing - tags: ["flb_stdout"] + tags: ["console"] polling_interval: 2000 enable_validator: false # By default, StringStamped message does not have a JSON schema since it uses custom input data ``` @@ -163,14 +169,15 @@ You will then need in another terminal to publish data on the input topic (`/hel ros2 topic pub -r 1 /hello-world dc_interfaces/msg/StringStamped "{data: '{\"hello\":\"world\"}'}" ``` -### Example 7: Custom ROS message to Stdout every time it is published +### Example 7: Custom ROS message to the console every time it is published ```yaml -destination_server: +dc_bridge: ros__parameters: - destination_plugins: ["flb_stdout"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" + destinations: ["console"] + console: + type: console + receives: records inputs: ["/dc/measurement/string_stamped"] measurement_server: @@ -180,7 +187,7 @@ measurement_server: plugin: "dc_measurements/StringStamped" topic_output: "/dc/measurement/my_string_stamped" topic: "/hello-world" - tags: ["flb_stdout"] + tags: ["console"] enable_validator: false timer_based: false # Get all data published on the input topic. Ignores polling_interval ``` diff --git a/doc/src/dc/data_pipeline.md b/doc/src/dc/data_pipeline.md index f154dc68b..680d85fc0 100644 --- a/doc/src/dc/data_pipeline.md +++ b/doc/src/dc/data_pipeline.md @@ -25,85 +25,3 @@ including a Linux parent-death signal, so Vector can never outlive the Bridge even across a SIGKILL/crash. Records published while the Bridge is down are dropped (topics are fire-and-forget); delivery resumes as soon as the respawned Bridge is ready. - -## Legacy (Humble): embedded Fluent Bit pipeline - -```mermaid -flowchart LR - dc_bringup_launch["DC Bringup Launch"] - yaml_param["YAML Parameter file"] - ros_middleware["ROS middleware"] - - subgraph m_s["Measurement Server"] - direction LR - meas_node["Measurement Node"] - measurement_plugins["Measurement plugins"] - meas_topics_out["ROS Topics"] - end - - subgraph d_s["Destination Server"] - direction LR - dest_node["Destination Node"] - dest_plugins["Destination plugins"] - end - - subgraph g_s["Group Server"] - direction LR - group_node["Group Node"] - group_grouped_data["Grouped data"] - group_topics_out["ROS Topics"] - end - - subgraph flb["Fluent Bit"] - direction LR - flb_server["Fluent Bit Server"] - flb_mem_buff["Memory buffer"] - flb_storage_buff["Storage buffer"] - flb_record["Record"] - flb_ros2_plugin["ROS2 plugin"] - flb_output["Outputs"] - end - - dc_bringup_launch--2.starts-->meas_node - dc_bringup_launch--3.starts-->dest_node - dc_bringup_launch--4.starts-->group_node - dc_bringup_launch--1.loads-->yaml_param - - meas_node--2.1.initializes-->measurement_plugins - measurement_plugins--"2.2.publish to"-->meas_topics_out - - dest_node--3.1.starts-->flb_server - dest_node--3.2.initializes-->dest_plugins - dest_node--3.3.loads-->flb_ros2_plugin - - flb_ros2_plugin--"3.3.subscribes to"-->meas_topics_out - flb_ros2_plugin--"3.3.subscribes to"-->group_topics_out - group_node--"4.1.subscribes to"-->meas_topics_out - group_node--4.2.generates-->group_grouped_data - group_grouped_data--"4.3.publishes to"-->group_topics_out - - flb_ros2_plugin--3.4.generates-->flb_record - flb_record--"3.5.stores in"-->flb_mem_buff - flb_record--"3.7.stores in"-->flb_storage_buff - flb_mem_buff--3.6.flushes-->flb_output - flb_storage_buff--3.8.flushes-->flb_output - - group_topics_out--"4.4.sends to"-->ros_middleware - meas_topics_out--"2.3.sends to"-->ros_middleware - ros_middleware--"when data received, sends to"-->flb_ros2_plugin -``` - -The data flow precised in this flowchart summarizes how data moves: -1. DC Bringup loads the yaml configuration file -2. DC Bringup starts the measurement node - 1. Measurement plugins are loaded and data starts to be collected - 2. Each plugin publishes what it collected on a ROS topic -3. DC Bringup starts the destination node - 1. Fluent Bit server is started: as soon as it receives records, it goes through filters and then is flushed to desired destinations - 2. Destination plugins are loaded - 3. ROS2 Fluent Bit plugin is loaded: it subscribes to measurement and group output topics - 4. Fluent Bit filters are initialized: it edits data received, modify the tags to match the desired output(s) and edit the timestamp field if required -4. DC Bringup starts the group node - 1. Subscribes to measurement and group topics outputs - 2. When all measurements of a group are received, it groups the JSONs into a new one - 3. It publishes the grouped data on another topic diff --git a/doc/src/dc/destinations.md b/doc/src/dc/destinations.md index 9d079892a..5ff38ef96 100644 --- a/doc/src/dc/destinations.md +++ b/doc/src/dc/destinations.md @@ -122,7 +122,7 @@ the remote key the Measurement is configured with: { "name": "map", "local_paths": { "yaml": "/tmp/map.yaml", "pgm": "/tmp/map.pgm" }, - "remote_paths": { "minio": { "yaml": "robot/map.yaml", "pgm": "robot/map.pgm" } } + "remote_paths": { "rustfs": { "yaml": "robot/map.yaml", "pgm": "robot/map.pgm" } } } ``` @@ -166,135 +166,3 @@ dc_bridge: # multipart_part_size_bytes: 8388608 # optional; >= 5 MiB for real S3 stores ``` ---- - -## Legacy (Humble): embedded Fluent Bit destination plugins - -Everything below documents the Humble-era architecture, which the DC 2.0 line -replaces (ADR-0001). - -## Description -A destination is where the data will be sent: AWS S3, stdout, AWS Kinesis. It has the possibility to use [outputs from fluentbit](https://docs.fluentbit.io/manual/pipeline/outputs). - -The destination node is similar to the measurement one. It: -* Either subscribes to data in the main node and data is forwarded with ros based destination plugins -* Either subscribes to data from the ros2 fluent bit plugin. - -The fluent bit plugin is the preferred one since when using it, we get all the benefits from fluent bit (especially data integrity). The ros2 fluent bit plugin uses rclc and is at the moment a fork of fluent bit. It could also be rewritten as a fluent bit plugin based on the GO interface, but pros and cons to move to it are not clear yet. - - -## Node parameters - -Destinations parameters are loaded dynamically. Here are the static ones: - -| Parameter name | Description | Type(s) | Default | -| --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | ----------------------- | -| [flb.flush](https://docs.fluentbit.io/manual/v/1.3/service) | Interval to flush output (seconds) | str | "1" | -| [flb.grace](https://docs.fluentbit.io/manual/v/1.3/service) | Wait time (seconds) on exit | str | "1" | -| [flb.log_level](https://docs.fluentbit.io/manual/v/1.3/service) | Diagnostic level (error/warning/info/debug/trace) | str | "info" | -| [flb.storage_path](https://docs.fluentbit.io/manual/administration/buffering-and-storage) | Set an optional location in the file system to store streams and chunks of data. If this parameter is not set, Input plugins can only use in-memory buffering. | str | "/var/log/flb-storage/" | -| [flb.storage_sync](https://docs.fluentbit.io/manual/administration/buffering-and-storage) | Configure the synchronization mode used to store the data into the file system. It can take the values normal or full. | str | "normal" | -| [flb.storage_checksum](https://docs.fluentbit.io/manual/administration/buffering-and-storage) | Enable the data integrity check when writing and reading data from the filesystem. The storage layer uses the CRC32 algorithm. | str | "off" | -| [flb.storage_backlog_mem_limit](https://docs.fluentbit.io/manual/administration/buffering-and-storage) | If storage.path is set, Fluent Bit will look for data chunks that were not delivered and are still in the storage layer, these are called backlog data. This option configure a hint of maximum value of memory to use when processing these records. | str | "5M" | -| [flb.scheduler_cap](https://docs.fluentbit.io/manual/administration/scheduling-and-retries) | Set a maximum retry time in seconds. The property is supported from v1.8.7. | str | "2000" | -| [flb.scheduler_base](https://docs.fluentbit.io/manual/administration/scheduling-and-retries) | Set a base of exponential backoff. The property is supported from v1.8.7. | str | "5" | -| [flb.http_server](https://docs.fluentbit.io/manual/administration/buffering-and-storage) | If true enable statistics HTTP server | bool | false | -| [flb.in_storage_type](https://docs.fluentbit.io/manual/administration/buffering-and-storage#input-section-configuration) | Specifies the buffering mechanism to use. It can be memory or filesystem. | str | "filesystem" | -| [flb.in_storage_pause_on_chunks_overlimit](https://docs.fluentbit.io/manual/administration/buffering-and-storage#input-section-configuration) | Specifies if file storage is to be paused when reaching the chunk limit. | str | "off" | - - -## Plugin parameters - -| Parameter name | Description | Type(s) | Default | -| -------------- | ------------------------------------------------- | ------------------------ | -------- | -| plugin | Plugin to load | list\[str\] | N/A | -| inputs | Topics to which to listen to get the data | list\[str\] | N/A | -| debug | Enable debug print | bool | false | -| time_format | Format the data will be printed | str("double", "iso8601") | "double" | -| time_key | Dictionary key from which date will be taken from | str | "date" | - -## Available plugins: - -| Name | Description | Source | -| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------ | -| [flb_file](./destinations/flb_file.md) | Write the data received through the input plugin to file. | [Fluent Bit File](https://docs.fluentbit.io/manual/pipeline/outputs/file) | -| [flb_files_metrics](./destinations/flb_files_metrics.md) | Ingest your records into the [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams/) service | This project, in fluent_bit_plugins | -| [flb_http](./destinations/flb_http.md) | JSON to http request via Fluent Bit | [Fluent Bit HTTP](https://docs.fluentbit.io/manual/pipeline/outputs/http) | -| [flb_kinesis_streams](./destinations/flb_kinesis_data_streams.md) | JSON to AWS Kinesis Streams via Fluent Bit | [Amazon Kinesis Data Streams](https://docs.fluentbit.io/manual/pipeline/outputs/kinesis) | -| [flb_minio](./destinations/flb_minio.md) | JSON to Minio via Fluent Bit | This project, in fluent_bit_plugins package | -| [flb_pgsql](./destinations/flb_pgsql.md) | JSON to PostgreSQL via Fluent Bit | [Fluent Bit PostgreSQL](https://docs.fluentbit.io/manual/pipeline/outputs/postgresql) | -| [flb_stdout](./destinations/flb_stdout.md) | JSON to STDOUT via Fluent Bit | [Fluent Bit Stdout](https://docs.fluentbit.io/manual/pipeline/outputs/standard-output) | -| [flb_s3](./destinations/flb_s3.md) | JSON to AWS S3 via Fluent Bit | [Fluent Bit S3](https://docs.fluentbit.io/manual/pipeline/outputs/s3) | -| [flb_tcp](./destinations/flb_tcp.md) | JSON to TCP via Fluent Bit | [Fluent Bit TCP & TLS](https://docs.fluentbit.io/manual/pipeline/outputs/tcp-and-tls) | -| [rcl](./destinations/rcl.md) | JSON to RCL | [ROS 2 logging](https://docs.ros.org/en/rolling/Tutorials/Demos/Logging-and-logger-configuration.html) | - - -## Example configuration - -```yaml -destination_server: - ros__parameters: - flb: - flush: "1" - flb_grace: "1" - log_level: "info" - storage_path: "/var/log/flb-storage/" - storage_sync: "full" - storage_checksum: "off" - storage_backlog_mem_limit: "1M" - scheduler_cap: "2000" - scheduler_base: "5" - metrics: true - in_storage_type: "filesystem" - in_storage_pause_on_chunks_overlimit: "off" - destination_plugins: ["flb_stdout"] - flb_minio: - plugin: "dc_destinations/FlbMinIO" - inputs: ["/dc/group/map"] - plugin_path: "/root/ws/src/ros2_data_collection/dc_destinations/flb_plugins/lib/out_minio.so" - endpoint: 127.0.0.1:9000 - access_key_id: HgJdDWeDQBiBWCwm - secret_access_key: plCMROO2VMZIKiqEwDd80dLJUCRvJ9iu - use_ssl: false - bucket: "mybucket" - src_fields: ["camera.local_img_paths.raw","camera.local_img_paths.rotated", "map.local_map_paths.yaml", "map.local_map_paths.pgm"] - upload_fields: ["camera.minio_img_paths.raw","camera.minio_img_paths.rotated", "map.minio_map_paths.yaml", "map.minio_map_paths.pgm"] - flb_pgsql: - plugin: "dc_destinations/FlbPgSQL" - inputs: ["/dc/group/memory_uptime"] - host: "127.0.0.1" - port: "5432" - user: fluentbit - password: password - database: "fluentbit" - table: "dc" - timestamp_key: "date" - async: false - time_format: "double" - time_key: "date" - rcl: - plugin: "dc_destinations/Rcl" - inputs: ["/dc/group/memory_uptime"] - flb_stdout: - plugin: "dc_destinations/FlbStdout" - inputs: ["/dc/group/cameras"] - inputs: ["/dc/group/cameras"] - time_format: "iso8601" - time_key: "date" - flb_http: - plugin: "dc_destinations/FlbHTTP" - inputs: ["/dc/group/memory_uptime"] - debug: true - flb_file: - plugin: "dc_destinations/FlbFile" - inputs: ["/dc/group/memory_uptime"] - path: "$HOME/data" - file: uptime - debug: false - throttle: - enable: false - rate: "1" - window: "5" - interval: "10s" - print_status: true -``` diff --git a/doc/src/dc/destinations/flb_file.md b/doc/src/dc/destinations/flb_file.md deleted file mode 100644 index 3d94ec949..000000000 --- a/doc/src/dc/destinations/flb_file.md +++ /dev/null @@ -1,30 +0,0 @@ -# File - Fluent Bit - -## Description - -The file output plugin allows to write the data received through the input plugin to file. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/file) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | ----------------- | -| **file** | Set file name to store the records. If not set, the file name will be the tag associated with the records. | str | N/A (Optional) | -| **format** | The format of the file content. See also [Format](https://docs.fluentbit.io/manual/pipeline/outputs/file#format) section. | str(out_file,plain,csv,ltsv) | "out_file" | -| **mkdir** | Recursively create output directory if it does not exist. Permissions set to 0755. | bool | true | -| **path** | Directory path to store files. If not set, Fluent Bit will write the files on it's own positioned directory. | str | "$HOME/data/" | -| **delimiter** | The character to separate each data. Default to ',' if format=csv, '\t'(TAB) if format=ltsv, '' else. | str | '' or ',' or '\t' | -| **label_delimiter** | The character to separate label and the value. Default: ':'. Used for ltsv. | str | '.' | - -## Node configuration - -```yaml -... -flb_file: - plugin: "dc_destinations/FlbFile" - inputs: ["/dc/group/cameras"] - file: "data" - format: "out_file" - mkdir: true - path: "$HOME/data/" -... -``` diff --git a/doc/src/dc/destinations/flb_files_metrics.md b/doc/src/dc/destinations/flb_files_metrics.md deleted file mode 100644 index 40cf5e51a..000000000 --- a/doc/src/dc/destinations/flb_files_metrics.md +++ /dev/null @@ -1,99 +0,0 @@ -# Files metrics - Fluent Bit - -## Description - -Tracks files (yaml files, jpg pictures etc.) sent to their destinations (s3, minio etc.). We can then delete files locally when they have been sent to all their storage destinations. - -It loads a Fluent Bit plugin we wrote ourselves to do this task, located in the fluent_bit_plugins package. - -It creates a table on the database (currently PostgreSQL only) to store each file status. - -```mermaid -erDiagram - files_metrics { - integer id - timestamp timestamp - text robot_name - text robot_id - text group_name - double duration - text local_path - text remote_path - bool uploaded - bool on_filesystem - bool deleted - bool ignored - text storage_type - text content_type - integer size - timestamp created_at - timestamp updated_at - } -``` - -## Parameters - -| Parameter | Description | Type | Default | -| --------------------------- | ----------------------------------------------- | ---------------------- | ---------------------------------------------------------------- | -| **db_type** | Database used. | str | "pgsql" | -| **file_storage** | Where data will be stored. | list\[str\](s3, minio) | N/A (Mandatory) | -| **delete_when_sent** | Delete file when sent. | bool | true | -| **minio.access_key_id** | Minio Access key ID. | str | N/A (Mandatory) | -| **minio.bucket** | Minio bucket name. | str | "dc_bucket" | -| **minio.endpoint** | Minio endpoint. | str | "127.0.0.1:9000" | -| **minio.secret_access_key** | Minio Secret access key. | str | N/A (Mandatory) | -| **minio.src_fields** | JSON fields containing local paths for Minio. | str | N/A (Mandatory) | -| **minio.upload_fields** | JSON fields containing remote paths for Minio. | str | N/A (Mandatory) | -| **minio.use_ssl** | Use SSL for Minio. | bool | true | -| **plugin_path** | Shared library path compiled by go. | str | "{package_share_directory}/flb_plugins/lib/out_files_metrics.so" | -| **pgsql.database** | Database name to connect to. | str | | -| **pgsql.host** | Hostname/IP address of the PostgreSQL instance. | str | "127.0.0.1" | -| **pgsql.password** | Password of PostgreSQL username. | str | "" | -| **pgsql.port** | PostgreSQL port. | str | "5432" | -| **pgsql.ssl** | Use ssl for PostgreSQL connection. | bool | true | -| **pgsql.table** | Table name where to store data. | str | "pg_table" | -| **pgsql.user** | PostgreSQL username. | str | | -| **plugin_path** | Shared library of the plugin. | str | /lib/out_files_metrics.so | -| **s3.access_key_id** | S3 Access key ID. | str | N/A (Mandatory) | -| **s3.bucket** | S3 bucket name. | str | N/A (Mandatory) | -| **s3.endpoint** | S3 endpoint. | str | N/A (Mandatory) | -| **s3.secret_access_key** | S3 Secret access key. | str | N/A (Mandatory) | -| **s3.src_fields** | JSON fields containing local paths for s3. | str | N/A (Mandatory) | -| **s3.upload_fields** | JSON fields containing remote paths for S3. | str | N/A (Mandatory) | - -## Example -```yaml -flb_files_metrics: - plugin: "dc_destinations/FlbFilesMetrics" - inputs: ["/dc/group/map"] - file_storage: ["minio", "s3"] - db_type: "pgsql" - debug: true - delete_when_sent: true - minio: - endpoint: 127.0.0.1:9000 - access_key_id: XEYqG4ZcPY5jiq5i - secret_access_key: ji011KCtI82ZeQS6UwsQAg8x9VR4lSaQ - use_ssl: false - bucket: "mybucket" - src_fields: ["map.local_paths.pgm", "map.local_paths.yaml"] - upload_fields: ["map.minio_paths.pgm", "map.minio_paths.yaml"] - s3: - endpoint: 127.0.0.1:9000 - access_key_id: XEYqG4ZcPY5jiq5i - secret_access_key: ji011KCtI82ZeQS6UwsQAg8x9VR4lSaQ - bucket: "mybucket" - src_fields: ["map.local_paths.yaml"] - upload_fields: ["map.s3_paths.yaml"] - pgsql: - host: "127.0.0.1" - port: "5432" - user: fluentbit - password: password - database: "fluentbit" - table: "files_metrics" - timestamp_key: "date" - time_format: "double" - time_key: "date" - ssl: false -``` diff --git a/doc/src/dc/destinations/flb_http.md b/doc/src/dc/destinations/flb_http.md deleted file mode 100644 index 710e10a2a..000000000 --- a/doc/src/dc/destinations/flb_http.md +++ /dev/null @@ -1,36 +0,0 @@ -# HTTP - Fluent Bit - -## Description - -The http output plugin allows to flush your records into a HTTP endpoint. For now the functionality is pretty basic and it issues a POST request with the data records in [MessagePack](http://msgpack.org/) (or JSON) format. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/http) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ----------- | -| **allow_duplicated_headers** | Specify if duplicated headers are allowed. If a duplicated header is found, the latest key/value set is preserved. | bool | true | -| **format** | Specify the data format to be used in the HTTP request body, by default it uses msgpack. Other supported formats are json, json_stream and json_lines and gelf. | str | "json" | -| **header** | Add a HTTP header key/value pair. Multiple headers can be set. | str | N/A | -| **headers_key** | Specify the key to use as the headers of the request (must prefix with "$"). The key must contain a map, which will have the contents merged on the request headers. This can be used for many purposes, such as specifying the content-type of the data contained in body_key. | str | N/A | -| **header_tag** | Specify an optional HTTP header field for the original message tag. | str | N/A | -| **http_user** | Basic Auth Username. | str | N/A | -| **http_passwd** | Basic Auth Password. Requires HTTP_User to be set. | str | N/A | -| **json_date_format** | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681). | str | "double" | -| **json_date_key** | Specify the name of the time key in the output record. To disable the time key just set the value to false. | str | "date" | -| **host** | IP address or hostname of the target HTTP Server. | str | "127.0.0.1" | -| **log_response_payload** | Specify if the response paylod should be logged or not. | str | true | -| **port** | TCP port of the target HTTP Server | str | "80" | -| **uri** | Specify an optional HTTP URI for the target web server, e.g: /something. | str | "/" | -## Node configuration - -```yaml -... -flb_http: - plugin: "dc_destinations/FlbHTTP" - inputs: ["/dc/measurement/data"] - host: "127.0.0.1" - port: 80 - uri: "/" - format: "json" -... -``` diff --git a/doc/src/dc/destinations/flb_influxdb.md b/doc/src/dc/destinations/flb_influxdb.md deleted file mode 100644 index 88733ca1e..000000000 --- a/doc/src/dc/destinations/flb_influxdb.md +++ /dev/null @@ -1,34 +0,0 @@ -# InfluxDB - Fluent Bit - -## Description - -The influxdb output plugin, allows to flush your records into a [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/) time series database. The following instructions assumes that you have a fully operational InfluxDB service running in your system. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/influxdb) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ---------------- | -------------------------------------------------------------------------------------------------------------- | ----------- | -------------- | -| **host** | IP address or hostname of the target InfluxDB service. | str | "127.0.01" | -| **port** | TCP port of the target InfluxDB service. | int | 8086 | -| **database** | InfluxDB database name where records will be inserted. | str | "fluentbit" | -| **bucket** | InfluxDB bucket name where records will be inserted - if specified, database is ignored and v2 of API is used. | str | N/A (Optional) | -| **org** | InfluxDB organization name where the bucket is (v2 only). | str | "fluent" | -| **sequence_tag** | The name of the tag whose value is incremented for the consecutive simultaneous events. | str | "_seq" | -| **http_user** | Optional username for HTTP Basic Authentication. | str | N/A (Optional) | -| **http_passwd** | Password for user defined in HTTP_User. | str | N/A (Optional) | -| **http_token** | Authentication token used with InfluDB v2 - if specified, both HTTP_User and HTTP_Passwd are ignored. | str | N/A (Optional) | -| **tag_keys** | List of keys that needs to be tagged. | list\[str\] | N/A (Optional) | -| **auto_tags** | Automatically tag keys where value is string. | bool | false | - -## Node configuration - -```yaml -... -flb_influxdb: - plugin: "dc_destinations/FlbInfluxDB" - inputs: ["/dc/measurement/uptime"] - host: "127.0.0.1" - port: 8086 - database: "ros" -... -``` diff --git a/doc/src/dc/destinations/flb_kinesis_data_streams.md b/doc/src/dc/destinations/flb_kinesis_data_streams.md deleted file mode 100644 index 2718d8610..000000000 --- a/doc/src/dc/destinations/flb_kinesis_data_streams.md +++ /dev/null @@ -1,32 +0,0 @@ -# AWS Kinesis Data Streams - Fluent Bit - -## Description - -The Amazon Kinesis Data Streams output plugin allows to ingest your records into the [Kinesis](https://aws.amazon.com/kinesis/data-streams/) service. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/kinesis) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ------------------- | -| **region** | The AWS region. | str | N/A | -| **stream** | The name of the Kinesis Streams Delivery stream that you want log records sent to. | str | N/A | -| **role_arn** | ARN of an IAM role to assume (for cross account access). | str | N/A | -| **time_key** | Add the timestamp to the record under this key. By default the timestamp from Fluent Bit will not be added to records sent to Kinesis. | str | N/A | -| **time_key_format** | strftime compliant format string for the timestamp; for example, the default is '%Y-%m-%dT%H:%M:%S'. Supports millisecond precision with '%3N' and supports nanosecond precision with '%9N' and '%L'; for example, adding '%3N' to support millisecond '%Y-%m-%dT%H:%M:%S.%3N'. This option is used with time_key. | str | "%Y-%m-%dT%H:%M:%S" | -| **log_key** | By default, the whole log record will be sent to Kinesis. If you specify a key name with this option, then only the value of that key will be sent to Kinesis. For example, if you are using the Fluentd Docker log driver, you can specify log_key log and only the log message will be sent to Kinesis. | str | N/A | -| **endpoint** | Specify a custom endpoint for the Kinesis API. | str | N/A | -| **auto_retry_requests** | Immediately retry failed requests to AWS services once. This option does not affect the normal Fluent Bit retry mechanism with backoff. Instead, it enables an immediate retry with no delay for networking errors, which may help improve throughput when there are transient/random networking issues. This option defaults to true. | bool | true | - -## Node configuration - -```yaml -... -flb_kinesis_streams: - plugin: "dc_destinations/FlbKinesisStreams" - inputs: ["/dc/measurement/data"] - region: "us-east-1" - stream: my_stream - time_key: "date" - time_key_format: "%Y-%m-%dT%H:%M:%S" -... -``` diff --git a/doc/src/dc/destinations/flb_minio.md b/doc/src/dc/destinations/flb_minio.md deleted file mode 100644 index 03feeb480..000000000 --- a/doc/src/dc/destinations/flb_minio.md +++ /dev/null @@ -1,80 +0,0 @@ -# Minio - Fluent Bit - -## Description - -The Minio Plugin allows to send files to Minio. It uses the GO interface provided by Fluent Bit. - - -## Difference with Fluent Bit S3 Plugin -The [S3 plugin](https://docs.fluentbit.io/manual/pipeline/outputs/s3) sends data as JSON to S3 or Minio but does not send files (e.g yaml, images etc.). - -## Parameters - -| Parameter | Description | Type | Default | -| --------------------- | ----------------------------------------------------------------------------------- | ----------- | -------------------------------------------------------- | -| **plugin_path** | Shared library path compiled by go. | str | "{package_share_directory}/flb_plugins/lib/out_minio.so" | -| **endpoint** | Endpoint for the S3 API. An endpoint can contain scheme and port. | str | "127.0.0.1:9000" | -| **access_key_id** | Access keys are long-term credentials for an IAM user or the AWS account root user. | str | N/A (mandatory) | -| **secret_access_key** | Secret-like password to connect along he access key id. | str | N/A (mandatory) | -| **use_ssl** | If set to true, https is used instead of http. Default is true. | str | "true" | -| **create_bucket** | Whether the bucket will be created. | str | "true" | -| **bucket** | Bucket name. | str | "dc_bucket" | -| **upload_fields** | Fields containing remote paths, separated by dots. | list\[str\] | N/A (mandatory) | -| **src_fields** | Fields containing local paths, separated by dots. | list\[str\] | N/A (mandatory) | - -## Node configuration - -```yaml -... -flb_minio: - verbose_plugin: false - time_format: "iso8601" - plugin: "dc_destinations/FlbMinIO" - inputs: ["/dc/measurement/camera", "/dc/measurement/map"] - endpoint: 127.0.0.1:9000 - access_key_id: XEYqG4ZcPY5jiq5i - secret_access_key: ji011KCtI82ZeQS6UwsQAg8x9VR4lSaQ - use_ssl: false - create_bucket: true - bucket: "mybucket" - src_fields: - [ - "local_paths.inspected" - "local_paths.minio.pgm" - "local_paths.minio.yaml" - ] - upload_fields: # Remote paths created by the measurement node configuration - [ - "remote_paths.minio.inspected", - "remote_paths.minio.pgm" - "remote_paths.minio.yaml" - ] -... -``` - -## Parameter handling - -```mermaid -flowchart TB - - subgraph ros2_plugin["Measurement plugin"] - direction LR - mp_src_fields["['local_paths.inspected', 'local_paths.minio.pgm', 'local_paths.minio.yaml']"] - mp_upload_fields["['remote_paths.inspected', 'remote_paths.minio.pgm', 'remote_paths.minio.yaml']"] - end - - subgraph flb_plugin["Fluent Bit plugin"] - direction LR - flb_src_fields["'local_paths.inspected, local_paths.minio.pgm, local_paths.minio.yaml'"] - flb_upload_fields["'remote_paths.inspected, remote_paths.minio.pgm, remote_paths.minio.yaml'"] - - flb_split_src_fields["['local_paths.inspected', 'local_paths.minio.pgm', 'local_paths.minio.yaml']"] - flb_split_upload_fields["['remote_paths.inspected', 'remote_paths.minio.pgm', 'remote_paths.minio.yaml']"] - end - - mp_src_fields--"to string"-->flb_src_fields - mp_upload_fields--"to string"-->flb_upload_fields - flb_src_fields--"to go array"-->flb_split_src_fields - flb_upload_fields--"to go array"-->flb_split_upload_fields - -``` diff --git a/doc/src/dc/destinations/flb_null.md b/doc/src/dc/destinations/flb_null.md deleted file mode 100644 index b92daaf4f..000000000 --- a/doc/src/dc/destinations/flb_null.md +++ /dev/null @@ -1,15 +0,0 @@ -# NULL - Fluent Bit - -## Description - -The null output plugin just throws away events. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/null) for more information. - -## Node configuration - -```yaml -... -flb_null: - plugin: "dc_destinations/FlbNull" - inputs: ["/dc/measurement/data"] -... -``` diff --git a/doc/src/dc/destinations/flb_pgsql.md b/doc/src/dc/destinations/flb_pgsql.md deleted file mode 100644 index 022af1ed5..000000000 --- a/doc/src/dc/destinations/flb_pgsql.md +++ /dev/null @@ -1,41 +0,0 @@ -# PostgreSQL - Fluent Bit - -## Description - -PostgreSQL is a very popular and versatile open source database management system that supports the SQL language and that is capable of storing both structured and unstructured data, such as JSON objects. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/postgresql) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ----------------- | -------------------------------------------------------------- | ---- | ------------ | -| **host** | Hostname/IP address of the PostgreSQL instance. | str | "127.0.0.1" | -| **port** | PostgreSQL port. | str | "5432" | -| **user** | PostgreSQL username. | str | "{username}" | -| **password** | Password of PostgreSQL username. | str | N/A | -| **database** | Database name to connect to. | str | "{username}" | -| **table** | Table name where to store data. | str | N/A | -| **timestamp_Key** | Key in the JSON object containing the record timestamp. | str | "date" | -| **async** | Define if we will use async or sync connections. | bool | false | -| **min_pool_size** | Minimum number of connection in async mode. | str | "1" | -| **max_pool_size** | Maximum amount of connections in async mode. | str | "4" | -| **cockroachdb** | Set to true if you will connect the plugin with a CockroachDB. | bool | false | - -## Node configuration - -```yaml -... -flb_pgsql: - plugin: "dc_destinations/FlbPgSQL" - inputs: ["/dc/group/data"] - host: "127.0.0.1" - port: 5432 - user: fluentbit - password: password - database: "fluentbit" - table: "dc" - timestamp_key: "date" - async: false - time_format: "double" - time_key: "date" -... -``` diff --git a/doc/src/dc/destinations/flb_s3.md b/doc/src/dc/destinations/flb_s3.md deleted file mode 100644 index ebeab59f1..000000000 --- a/doc/src/dc/destinations/flb_s3.md +++ /dev/null @@ -1,52 +0,0 @@ -# AWS S3 - Fluent Bit - -## Description - -The Amazon S3 output plugin allows to ingest your records into the [S3](https://aws.amazon.com/s3/) service. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/s3) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ----------------------------------------- | -| **region** | The AWS region of your S3 bucket. | str | "us-east-1" | -| **bucket** | S3 Bucket name. | str | N/A (Mandatory) | -| **json_date_key** | Specify the name of the time key in the output record. To disable the time key just set the value to false. | str | "date" | -| **json_date_format** | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681). | str | "iso8601" | -| **total_file_size** | Specifies the size of files in S3. Maximum size is 50G, minimim is 1M. | str | "100M" | -| **upload_chunk_size** | The size of each 'part' for multipart uploads. Max: 50M. | str | "50M" | -| **upload_timeout** | Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. | str | "10m" | -| **store_dir** | Directory to locally buffer data before sending. When multipart uploads are used, data will only be buffered until the upload_chunk_size is reached. S3 will also store metadata about in progress multipart uploads in this directory; this allows pending uploads to be completed even if Fluent Bit stops and restarts. It will also store the current $INDEX value if enabled in the S3 key format so that the $INDEX can keep incrementing from its previous value after Fluent Bit restarts. | str | "/tmp/fluent-bit/s3" | -| **store_dir_limit_size** | The size of the limitation for disk usage in S3. Limit the amount of s3 buffers in the store_dir to limit disk usage. Note: Use store_dir_limit_size instead of storage.total_limit_size which can be used to other plugins, because S3 has its own buffering system. | int | 0 | -| **s3_key_format** | Format string for keys in S3. This option supports a UUID, strftime time formatters, a syntax for selecting parts of the Fluent log tag using a syntax inspired by the rewrite_tag filter. Add $UUID in the format string to insert a random string. Add $INDEX in the format string to insert an integer that increments each upload. The $INDEX value will be saved in the store_dir so that if Fluent Bit restarts the value will keep incrementing from the previous run. Add $TAG in the format string to insert the full log tag; add $TAG\[0\] to insert the first part of the tag in the s3 key. The tag is split into “parts” using the characters specified with the s3_key_format_tag_delimiters option. Add extension directly after the last piece of the format string to insert a key suffix. If you want to specify a key suffix and you are in use_put_object mode, you must specify $UUID as well. More explanations can be found in the S3 Key Format explainer section further down in this document. See the in depth examples and tutorial in the documentation. Time in s3_key is the timestamp of the first record in the S3 file. | str | "/fluent-bit-logs/$TAG/%Y/%m/%d/%H/%M/%S" | -| **static_file_path** | Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. | bool | false | -| **s3_key_format_tag_delimiters** | A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. See the in depth examples and tutorial in the documentation. | str | N/A | -| **use_put_object** | Use the S3 PutObject API, instead of the multipart upload API. When this option is on, key extension is only available when $UUID is specified in s3_key_format. If $UUID is not included, a random string will be appended at the end of the format string and the key extension cannot be customized in this case. | bool | false | -| **role_arn** | ARN of an IAM role to assume (ex. for cross account access). | str | N/A | -| **endpoint** | Custom endpoint for the S3 API. An endpoint can contain scheme and port. | str | N/A | -| **sts_endpoint** | Custom endpoint for the STS API. | str | N/A | -| **canned_acl** | [Predefined Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) policy. | str | N/A | -| **compression** | Compression type for S3 objects. 'gzip' is currently the only supported value by default. If Apache Arrow support was enabled at compile time, you can also use 'arrow'. For gzip compression, the Content-Encoding HTTP Header will be set to 'gzip'. Gzip compression can be enabled when use_put_object is 'on' or 'off' (PutObject and Multipart). Arrow compression can only be enabled with use_put_object On. | str | N/A | -| **content_type** | A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. | str | N/A | -| **send_content_md5** | Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. | bool | false | -| **auto_retry_requests** | Immediately retry failed requests to AWS services once. This option does not affect the normal Fluent Bit retry mechanism with backoff. Instead, it enables an immediate retry with no delay for networking errors, which may help improve throughput when there are transient/random networking issues. | bool | true | -| **log_key** | By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. For example, if you are using Docker, you can specify log_key log and only the log message will be sent to S3. | str | N/A | -| **preserve_data_ordering** | Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. | bool | true | -| **storage_class** | Specify the [storage class](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass) for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. | str | N/A | -| **retry_limit** | Integer value to set the maximum number of retries allowed. For previous version, the number of retries is 5 and is not configurable. | int | 1 | -| **external_id** | Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. | str | N/A | - -## Node configuration - -```yaml -... -flb_s3: - plugin: "dc_destinations/FlbS3" - inputs: ["/dc/group/data"] - bucket: my-bucket - region: us-west-2 - total_file_size: "50M" - use_put_object: false - compression: "gzip" - s3_key_format: "/$TAG/%Y/%m/%d/%H_%M_%S.gz" -... -``` diff --git a/doc/src/dc/destinations/flb_slack.md b/doc/src/dc/destinations/flb_slack.md deleted file mode 100644 index 0e1115327..000000000 --- a/doc/src/dc/destinations/flb_slack.md +++ /dev/null @@ -1,22 +0,0 @@ -# Slack - Fluent Bit - -## Description - -The Slack output plugin delivers records or messages to your preferred Slack channel. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/slack) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| ----------- | -------------------------------------------------- | ---- | --------------- | -| **webhook** | Absolute address of the Webhook provided by Slack. | str | N/A (Mandatory) | - -## Node configuration - -```yaml -... -flb_slack: - plugin: "dc_destinations/FlbSlack" - inputs: ["/dc/group/data"] - webhook: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX -... -``` diff --git a/doc/src/dc/destinations/flb_stdout.md b/doc/src/dc/destinations/flb_stdout.md deleted file mode 100644 index 704f07004..000000000 --- a/doc/src/dc/destinations/flb_stdout.md +++ /dev/null @@ -1,26 +0,0 @@ -# File - Fluent Bit - -## Description - -The stdout output plugin allows to print to the standard output the data received through the input plugin. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/standard-output) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | -------- | -| **format** | Specify the data format to be printed. Supported formats are msgpack json, json_lines and json_stream. | str | "json" | -| **json_date_key** | Specify the name of the time key in the output record. To disable the time key just set the value to false. | str | "date" | -| **json_date_format** | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681). | str | "double" | - -## Node configuration - -```yaml -... -flb_stdout: - plugin: "dc_destinations/FlbStdout" - inputs: ["/dc/group/data"] - format: "json" - json_date_key: "date" - json_date_format: "iso8601" -... -``` diff --git a/doc/src/dc/destinations/flb_tcp.md b/doc/src/dc/destinations/flb_tcp.md deleted file mode 100644 index 11e2b5bd5..000000000 --- a/doc/src/dc/destinations/flb_tcp.md +++ /dev/null @@ -1,35 +0,0 @@ -# TCP & TLS - Fluent Bit - -## Description - -The tcp output plugin allows to send records to a remote TCP server. The payload can be formatted in different ways as required.. See [fluent bit page](https://docs.fluentbit.io/manual/pipeline/outputs/tcp-and-tls) for more information. - -## Parameters - -| Parameter | Description | Type | Default | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | -------------- | -| **host** | Target host where Fluent-Bit or Fluentd are listening for Forward messages. | str | "127.0.0.1" | -| **port** | TCP Port of the target service. | int(>0 <65536) | 5170 | -| **format** | Specify the data format to be printed. Supported formats are msgpack json, json_lines and json_stream. | str | "msgpack" | -| **json_date_key** | Specify the name of the time key in the output record. To disable the time key just set the value to false. | str | "date" | -| **json_date_format** | Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681) | str | "double" | -| **workers** | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | int | 2 | -| **tls.active** | Enable or disable TLS support. | bool | false | -| **tls.verify** | Force certificate validation. | bool | true | -| **tls.debug** | Set TLS debug verbosity level. It accept the following values: 0 (No debug), 1 (Error), 2 (State change), 3 (Informational) and 4 Verbose. | int(>=0 <=4) | 1 | -| **tls.ca_file** | Absolute path to CA certificate file. | str | N/A (Optional) | -| **tls.crt_file** | Absolute path to Certificate file. | str | N/A (Optional) | -| **tls.key_file** | Absolute path to private Key file. | str | N/A (Optional) | -| **tls.key_passwd** | Optional password for tls.key_file file. | str | N/A (Optional) | - -## Node configuration - -```yaml -... -flb_tcp: - plugin: "dc_destinations/FlbTCP" - inputs: ["/dc/measurement/uptime"] - host: "127.0.0.1" - port: 5170 -... -``` diff --git a/doc/src/dc/destinations/rcl.md b/doc/src/dc/destinations/rcl.md deleted file mode 100644 index a00fa8d14..000000000 --- a/doc/src/dc/destinations/rcl.md +++ /dev/null @@ -1,8 +0,0 @@ -# RCL - RCLCPP - -## Description - -The RCL output collects the data with ROS and display it with `RCLCPP_INFO` call. - -## Warning -Note that plugins which are not using Fluent Bit like this one don't have any data integrity and persistence. Do not use in production! diff --git a/doc/src/dc/faq.md b/doc/src/dc/faq.md index 254a26039..1acaa6a4b 100644 --- a/doc/src/dc/faq.md +++ b/doc/src/dc/faq.md @@ -6,45 +6,23 @@ Even though measurements and destinations will constantly be added in the future Create a feature request in [Github Discussions](https://github.com/Minipada/ros2_data_collection/discussions/categories/ideas-and-feature-requests)...or better, write your plugin and do a Pull Request. -## Can I use DC without Fluent Bit backend? -It is possible to use DC without Fluent Bit as backend. There is an example with the RCL destination, which subscribes to a ROS topic and prints it in the console. - -Still, a lot of work would be required to disable Fluent Bit and keep similar features (backpressure handling, data persistence and measurements and destinations available) - -## How can I add a Fluent Bit plugin? - -Currently, from my knowledge, it is only possible to load a plugin using the C api which loads a shared library. Because of that, I expect more languages can be used, such as Rust, the only requirement is to use the bindings. For measurements, it is not a problem, we can write a node in any language publishing on a StringStamped message and use the measurement plugin with the same name. - -Destination plugins can be written in C or Go. You can find examples in the fluent_bit_plugins package. - -## How can I use a Python API with Fluent Bit - -Adding a destination plugin which uses a Python library is not cleared. We have not yet solved this problem and would welcome one, since many API libraries are available in this language. - -Note for the courageous ones: I tried once to create python-C bindings of Fluent Bit, by creating .h files like in the [fluent-bit-go](https://github.com/fluent/fluent-bit-go) and use ctypesgen(https://github.com/ctypesgen/ctypesgen), like this: - -```bash -ctypesgen \ - --allow-gnu-c \ - --output-language=py32 \ - ~/ws/fluent-bit/include/fluent-bit.h \ - -I./lib/msgpack-c/include/ \ - -I./lib/monkey/include \ - -I./build/lib/monkey/include/monkey \ - -I./lib/cfl/include \ - -I./lib/cfl/lib/xxhash \ - -I./include \ - -I./lib/flb_libco \ - -I./lib/c-ares-1.19.0/include \ - -I./lib/cmetrics/include \ - -I./lib/ctraces/include \ - -I./lib/ctraces/lib/mpack/src \ - -o fluent_bit.py -``` - -It creates a base for the bindings. While I suppose it is possible to generate them, I'm puzzled as how this will be loaded by Fluent Bit afterwards. - -Whatever you write, feel free to make a Pull Request and share your work! +## How can I send data to a Destination that isn't blessed? + +The Bridge (`dc_bridge`) renders its Shipper's (Vector) config from plain ROS +parameters for a **blessed set** of Destination types only (PostgreSQL, +S3-compatible storage, file, console — see [Destinations](./destinations.md)). Every +other sink in [Vector's catalog](https://vector.dev/docs/reference/configuration/sinks/) +(Kafka, Kinesis, InfluxDB, webhooks, …) is reachable through the **passthrough**: list a +raw Vector config snippet (TOML) in the `custom_config_files` parameter, and consume the +public `dc.` route it needs. No DC code, plugin, or extra language required — only +Vector configuration. + +## Can a passthrough snippet be generated or written in a language other than TOML? + +The snippet the Bridge merges in must be Vector's own TOML configuration syntax — DC +does not transform it. If you would rather generate that TOML from another language or +tool, nothing stops you from doing so as a build or deploy step; the Bridge only reads +the resulting file. ## My group data is not published on the group topic diff --git a/doc/src/dc/groups.md b/doc/src/dc/groups.md index e5a83af98..c6aa9530e 100644 --- a/doc/src/dc/groups.md +++ b/doc/src/dc/groups.md @@ -42,23 +42,23 @@ group_server: output: "/dc/group/memory_cpu" sync_delay: 5.0 group_key: "memory_cpu" - tags: ["flb_pgsql"] + tags: ["pgsql"] memory_uptime: inputs: ["/dc/measurement/memory", "/dc/measurement/uptime"] output: "/dc/group/memory_uptime" sync_delay: 5.0 group_key: "memory_uptime" - tags: ["flb_stdout"] + tags: ["console"] cameras: inputs: ["/dc/measurement/camera"] output: "/dc/group/cameras" sync_delay: 5.0 group_key: "cameras" - tags: ["flb_minio", "flb_stdout"] + tags: ["rustfs", "console"] map: inputs: ["/dc/measurement/map"] output: "/dc/group/map" sync_delay: 5.0 group_key: "map" - tags: ["flb_minio", "flb_stdout"] + tags: ["rustfs", "console"] ``` diff --git a/doc/src/dc/introduction.md b/doc/src/dc/introduction.md index 1f4f7eff6..5d7ee132a 100644 --- a/doc/src/dc/introduction.md +++ b/doc/src/dc/introduction.md @@ -45,7 +45,7 @@ For detailed instructions: The DC (Data Collection) project aims at integrating data collection pipelines into ROS 2. The goal is to integrate data collection pipelines with existing APIs to enable data analytics, rather than live monitoring, which already has excellent tools available. As companies increasingly turn to autonomous robots, the ability to understand and improve operations for any type of machine in any environment has become crucial. This involves mostly pick and drop and inspection operations. This framework aims at helping collecting, validating (through JSON schemas) and sending reliably the data to create such APIs and dashboards. -DC uses a modular approach, based on [pluginlib](https://index.ros.org/p/pluginlib/) and greatly inspired by [Nav2](https://navigation.ros.org/) for its architecture. Pluginlib is used to configure which measurements are collected and where the data goes. Measurements and destinations are pluginlib plugins. In addition to pluginlib, most plugins use [Fluent Bit](https://fluentbit.io/) in the backend: *Fluent Bit is a super fast, lightweight, and highly scalable logging and metrics processor and forwarder. It is the preferred choice for cloud and containerized environments. Developed and interfaced in C, it has already many features we directly can use, especially: high performance, reliability and data integrity (backpressure handling and data buffering in memory and filesystem)*. +DC uses a modular approach, based on [pluginlib](https://index.ros.org/p/pluginlib/) and greatly inspired by [Nav2](https://navigation.ros.org/) for its architecture. Pluginlib is used to configure which measurements are collected. Measurements are pluginlib plugins. Data leaves the robot through the Bridge (`dc_bridge`), a thin ROS 2 node that renders and supervises an external Shipper, [Vector](https://vector.dev/): *Vector is a fast, lightweight observability data pipeline, distributed as a single static binary, with native sinks for PostgreSQL, S3-compatible storage, and many more. DC gets its performance, reliability, and data integrity (backpressure handling and disk buffering) without embedding or forking it*. ## Why collect data from robots? @@ -70,16 +70,16 @@ DC uses a modular approach, based on [pluginlib](https://index.ros.org/p/pluginl * **Trigger-based data collection**: collect data when a defined set of combination of all, any, or no condition are met * **Customizable record collection**: configure the number of records to collect at the start and when a condition is activated. * **Data inspection**: inspect data from camera input including barcode and QR codes -* **Fast and efficient**: high performance, using Fluent Bit for backend processing, and designed to minimize code duplication and reduce human errors +* **Fast and efficient**: high performance, using an external Vector shipper for backend processing, and designed to minimize code duplication and reduce human errors * **Grouped measurements**: measurements can be grouped using the group node based on the ApproximateTimeSynchronizer * **File saving**: files can be saved, including map_server maps, camera images, and any file produced by a measurement * **Easy to use**: designed to be easy to learn and use * **No C++ 3rd party library required**: all 3rd party libraries have a vendor package in the repository -And inherited from Fluent Bit: +And inherited from the Vector shipper: -* [Backpressure handling](https://docs.fluentbit.io/manual/v/1.0/configuration/backpressure) -* [Data buffering in memory and filesystem](https://docs.fluentbit.io/manual/v/1.0/configuration/buffering) +* Backpressure handling +* [Disk buffering](https://vector.dev/docs/reference/configuration/global-options/#data_dir), persisting Records across Destination outages and reboots Here is an example of a pipeline: @@ -130,7 +130,7 @@ flowchart LR subgraph d_n["Destination node"] pl_pgsql["PostgreSQL"] - pl_minio["Minio"] + pl_rustfs["RustFS"] pl_s3["S3"] end @@ -151,8 +151,8 @@ flowchart LR gr_boot_system -- os, network interfaces\n, permissions and uptime --> pl_pgsql gr_robot -- Robot cmd_vel, position. speed --> pl_pgsql gr_system -- Available space,\n memory used and cpu usage --> pl_pgsql - gr_inspection -- Image paths on s3 and minio --> pl_pgsql - gr_inspection -- Raw, rotated and/or inspected images --> pl_minio + gr_inspection -- Image paths on s3 and rustfs --> pl_pgsql + gr_inspection -- Raw, rotated and/or inspected images --> pl_rustfs gr_inspection -- Raw, rotated and/or inspected images --> pl_s3 ``` diff --git a/doc/src/dc/measurements.md b/doc/src/dc/measurements.md index 355916c44..1c5b732f5 100644 --- a/doc/src/dc/measurements.md +++ b/doc/src/dc/measurements.md @@ -16,8 +16,7 @@ Measurements are a single data unit presented in JSON format, that can contain d ## Node parameters -The node starts the fluent bit engine and its ros2 plugin and enables data collection from ROS 2 topics. This plugin will subscribe to the configured ROS 2 topics and data will be collected by Fluent Bit to destinations enabled by the [destination node](./destinations.md). -Each topic is configured in a measurement, which is loaded in this node with pluginlib. +This node enables data collection from ROS 2 topics. Each topic is configured in a measurement, which is loaded in this node with pluginlib; the Bridge (`dc_bridge`) subscribes to the same topics and forwards Records to the destinations enabled there (see [Destinations](./destinations.md)). In addition, conditions are pluginlibs plugin also loaded dynamically. They are optional plugins that allow to collect on some conditions, e.g robot is moving. | Parameter name | Description | Type(s) | Default | @@ -25,7 +24,7 @@ In addition, conditions are pluginlibs plugin also loaded dynamically. They are | measurement_plugins | Name of the measurement plugins to load | list\[str\] | N/A (mandatory) | | condition_plugins | Name of the condition plugins to load | list\[str\] | N/A (mandatory) | | save_local_base_path | Path where files will be saved locally (e.g camera images). Expands $X to environment variables and =Y to custom string parameters | str | "$HOME/ros2/data/%Y/%M/%D/%H" | -| all_base_path | Path where files will be saved at their destination (S3, minio...). Expands $X to environment variables and =Y to custom string parameters | str | "" | +| all_base_path | Path where files will be saved at their destination (S3, RustFS...). Expands $X to environment variables and =Y to custom string parameters | str | "" | | custom_str_params_list | Custom strings to use in other parameters. They are also appended in the json sent to the destination | list\[str\] | N/A | | custom_str_params.force_override | Override values if the keys are already present in the measurement. Applies to all and can be overridden by `custom_str_params..force_override` | bool | false | | | @@ -54,7 +53,7 @@ Each measurement is collected through a node and has these configuration paramet | **condition_max_measurements** | Collect a maximum of n measurements when conditions are activated (-1 = never, 0 = infinite) | int | 0 | | **enable_validator** | Will validate the data against a JSON schema | bool | true | | **json_schema_path** | Path to the JSON schema, ignored if empty string | str | N/A (optional) | -| **tags** | Tags used by Fluent Bit to do the matching to destinations | list\[str\] | N/A (mandatory) | +| **tags** | Destination names, used by the Bridge to match Records to destinations | list\[str\] | N/A (mandatory) | | **remote_prefixes** | Prefixes to apply to the paths when sending files to a destination | str | N/A (optional) | | **remote_keys** | Used by some plugins to generate remote paths | list\[str\] | N/A (optional) | | **if_all_conditions** | Collect only if all conditions are activated | list\[str\] | N/A (optional) | diff --git a/doc/src/dc/measurements/camera.md b/doc/src/dc/measurements/camera.md index 57ee52948..9c4331090 100644 --- a/doc/src/dc/measurements/camera.md +++ b/doc/src/dc/measurements/camera.md @@ -24,7 +24,7 @@ Save camera image files: raw, rotated and/or inspected. Images can be inspected | **save_rotated_path** | Path to save the rotated camera image. Expands environment variables and datetime format are expanded | str | "camera/rotated/%Y-%m-%dT%H:%M:%S" | ## Measurement node configuration -The remote paths are also saved in the JSON under *._img_paths.(raw|rotated|inspected)*. If images want to be sent to Minio, add "minio" in *remote_keys*. This will add a remote path that can later be used in your API. +The remote paths are also saved in the JSON under *._img_paths.(raw|rotated|inspected)*. If images want to be sent to a self-hosted S3-compatible store such as [RustFS](https://rustfs.com/), add "rustfs" in *remote_keys*. This will add a remote path that can later be used in your API. Note that this remote key is not included in the JSON schema, which only contains the local paths. If you want to enforce the schema with your custom remote key, you will need to write it and load it manually. @@ -50,36 +50,30 @@ camera: rotation_angle: 0 detection_modules: ["barcode"] remote_prefixes: [""] - remote_keys: ["minio"] # Will create paths for Minio, does not send the file + remote_keys: ["rustfs"] # Will create paths for RustFS, does not send the file ``` -### Destination node configuration -Now that the path is set, it can be used to know where to send the image: +### Destination (dc_bridge) configuration +Now that the path is set, it can be used to know where to send the image. The +Destination name (`rustfs`) must match the `remote_keys` entry above — the Uploader +matches a Record's `remote_paths` keys against `receives: files` Destination names (see +[Destinations](../destinations.md)): ```yaml -... -flb_minio: - verbose_plugin: false - time_format: "iso8601" - plugin: "dc_destinations/FlbMinIO" - inputs: ["/dc/group/cameras"] - endpoint: 127.0.0.1:9000 - access_key_id: XEYqG4ZcPY5jiq5i - secret_access_key: ji011KCtI82ZeQS6UwsQAg8x9VR4lSaQ - use_ssl: false - create_bucket: true - bucket: "mybucket" - src_fields: - [ - "camera.local_img_paths.raw", - "camera.local_img_paths.inspected" - ] - upload_fields: # Remote paths created by the measurement node configuration - [ - "camera.minio_img_paths.raw", - "camera.minio_img_paths.inspected" - ] -... +dc_bridge: + ros__parameters: + destinations: ["rustfs", "pgsql"] + rustfs: + type: s3 + receives: files + inputs: ["/dc/group/cameras"] + endpoint: "http://127.0.0.1:9000" + access_key_id: "XEYqG4ZcPY5jiq5i" + secret_access_key: "ji011KCtI82ZeQS6UwsQAg8x9VR4lSaQ" + force_path_style: true + bucket: "mybucket" + files: + metadata_destination: "pgsql" # a receives: records Destination for status rows ``` ## Schema diff --git a/doc/src/dc/measurements/cpu.md b/doc/src/dc/measurements/cpu.md index f2f9cc945..148b66d7e 100644 --- a/doc/src/dc/measurements/cpu.md +++ b/doc/src/dc/measurements/cpu.md @@ -80,7 +80,7 @@ Collect cpu usage: average cpu, number of processes running and processes sorted cpu: plugin: "dc_measurements/Cpu" topic_output: "/dc/measurement/cpu" - tags: ["flb_stdout"] + tags: ["console"] max_processes: 10 cpu_min: 10.0 ``` diff --git a/doc/src/dc/measurements/distance_traveled.md b/doc/src/dc/measurements/distance_traveled.md index fb2ed48ea..a5831a065 100644 --- a/doc/src/dc/measurements/distance_traveled.md +++ b/doc/src/dc/measurements/distance_traveled.md @@ -36,7 +36,7 @@ Collect total distance traveled in the robot since it is powered. distance_traveled: plugin: "dc_measurements/DistanceTraveled" topic_output: "/dc/measurement/distance_traveled" - tags: ["flb_stdout"] + tags: ["console"] global_frame: "map" robot_base_frame: "base_link" transform_timeout: 0.1 diff --git a/doc/src/dc/measurements/dummy.md b/doc/src/dc/measurements/dummy.md index 9e590fc31..8e54d87aa 100644 --- a/doc/src/dc/measurements/dummy.md +++ b/doc/src/dc/measurements/dummy.md @@ -34,5 +34,5 @@ The dummy measurement, generates dummy events. It is useful for testing, debuggi dummy: plugin: "dc_measurements/Dummy" topic_output: "/dc/measurement/dummy" - tags: ["flb_stdout"] + tags: ["console"] ``` diff --git a/doc/src/dc/measurements/map.md b/doc/src/dc/measurements/map.md index 6a3543be2..bfb0094af 100644 --- a/doc/src/dc/measurements/map.md +++ b/doc/src/dc/measurements/map.md @@ -95,7 +95,7 @@ Save map using nav2_map_server and collect the map of the local map saved. The m map: plugin: "dc_measurements/Map" topic_output: "/dc/measurement/map" - tags: ["flb_stdout"] + tags: ["console"] topic: "/map" save_map: "map/%Y-%m-%dT%H:%M:%S" save_map_timeout: 0.2 diff --git a/doc/src/dc/measurements/memory.md b/doc/src/dc/measurements/memory.md index 71b6bf1e3..d7b9bd38a 100644 --- a/doc/src/dc/measurements/memory.md +++ b/doc/src/dc/measurements/memory.md @@ -29,5 +29,5 @@ Collect memory used in percentage. memory: plugin: "dc_measurements/Memory" topic_output: "/dc/measurement/memory" - tags: ["flb_stdout"] + tags: ["console"] ``` diff --git a/doc/src/dc/measurements/network.md b/doc/src/dc/measurements/network.md index a788e9694..64cc3426c 100644 --- a/doc/src/dc/measurements/network.md +++ b/doc/src/dc/measurements/network.md @@ -47,7 +47,7 @@ Collects ping value, whether or not the PC is online and interfaces available. network: plugin: "dc_measurements/Network" topic_output: "/dc/measurement/network" - tags: ["flb_stdout"] + tags: ["console"] ping_address: 192.168.0.1 ping_timeout: 500 ``` diff --git a/doc/src/dc/measurements/os.md b/doc/src/dc/measurements/os.md index 7a122b657..49798a355 100644 --- a/doc/src/dc/measurements/os.md +++ b/doc/src/dc/measurements/os.md @@ -41,5 +41,5 @@ Collects the Operating System information: cpus, operating system name and kerne os: plugin: "dc_measurements/OS" topic_output: "/dc/measurement/os" - tags: ["flb_stdout"] + tags: ["console"] ``` diff --git a/doc/src/dc/measurements/permissions.md b/doc/src/dc/measurements/permissions.md index 46a81b6df..8df03afb8 100644 --- a/doc/src/dc/measurements/permissions.md +++ b/doc/src/dc/measurements/permissions.md @@ -47,7 +47,7 @@ Collect UID, GID, if a file or directory exists and its permissions (in rwx or i permission_home_dc: plugin: "dc_measurements/Permissions" topic_output: "/dc/measurement/permissions_home_dc" - tags: ["flb_stdout"] + tags: ["console"] path: "$HOME/dc" format: "rwx" ``` diff --git a/doc/src/dc/measurements/position.md b/doc/src/dc/measurements/position.md index ad6fb2f83..02a475175 100644 --- a/doc/src/dc/measurements/position.md +++ b/doc/src/dc/measurements/position.md @@ -44,5 +44,5 @@ Collect x, y and yaw of the robot. position: plugin: "dc_measurements/Position" topic_output: "/dc/measurement/position" - tags: ["flb_stdout"] + tags: ["console"] ``` diff --git a/doc/src/dc/measurements/speed.md b/doc/src/dc/measurements/speed.md index c18809414..e4b609f51 100644 --- a/doc/src/dc/measurements/speed.md +++ b/doc/src/dc/measurements/speed.md @@ -67,6 +67,6 @@ Collect robot speed using the Odom topic. speed: plugin: "dc_measurements/Speed" topic_output: "/dc/measurement/speed" - tags: ["flb_stdout"] + tags: ["console"] odom_topic: "/odom" ``` diff --git a/doc/src/dc/measurements/storage.md b/doc/src/dc/measurements/storage.md index eccc09d9d..97d97b84c 100644 --- a/doc/src/dc/measurements/storage.md +++ b/doc/src/dc/measurements/storage.md @@ -45,6 +45,6 @@ Collect storage information on a directory. storage_home: plugin: "dc_measurements/Storage" topic_output: "/dc/measurement/storage_home" - tags: ["flb_stdout"] + tags: ["console"] path: "$HOME" ``` diff --git a/doc/src/dc/measurements/string_stamped.md b/doc/src/dc/measurements/string_stamped.md index 5b2ce1c88..420e9066c 100644 --- a/doc/src/dc/measurements/string_stamped.md +++ b/doc/src/dc/measurements/string_stamped.md @@ -23,7 +23,7 @@ Given that the data is customized here, there is no default schema. my_data: plugin: "dc_measurements/StringStamped" topic_output: "/dc/measurement/my_data" - tags: ["flb_stdout"] + tags: ["console"] topic: "/hello-world" timer_based: true ``` diff --git a/doc/src/dc/measurements/tcp_health.md b/doc/src/dc/measurements/tcp_health.md index 3ce2b741b..0d2cf230c 100644 --- a/doc/src/dc/measurements/tcp_health.md +++ b/doc/src/dc/measurements/tcp_health.md @@ -44,10 +44,10 @@ Collects status of a TCP server. ... tcp_health: plugin: "dc_measurements/TCPHealth" - topic_output: "/dc/measurement/minio_health" - group_key: "minio_health" - tags: ["flb_stdout"] + topic_output: "/dc/measurement/rustfs_health" + group_key: "rustfs_health" + tags: ["console"] host: "127.0.0.1" port: 9000 - name: "minio_api" + name: "rustfs_api" ``` diff --git a/doc/src/dc/measurements/uptime.md b/doc/src/dc/measurements/uptime.md index 000b4ef28..2ceb3456c 100644 --- a/doc/src/dc/measurements/uptime.md +++ b/doc/src/dc/measurements/uptime.md @@ -28,5 +28,5 @@ Time since when the robot PC has been on. uptime: plugin: "dc_measurements/Uptime" topic_output: "/dc/measurement/uptime" - tags: ["flb_stdout"] + tags: ["console"] ``` diff --git a/fluent_bit_plugins/CMakeLists.txt b/fluent_bit_plugins/CMakeLists.txt deleted file mode 100644 index 396f7ce8f..000000000 --- a/fluent_bit_plugins/CMakeLists.txt +++ /dev/null @@ -1,59 +0,0 @@ -cmake_minimum_required(VERSION 3.5) -project(fluent_bit_plugins) - -set(dependencies - dc_interfaces - fluent_bit_vendor - rcl - rclc -) - -find_package(ament_cmake REQUIRED) -foreach(Dependency IN ITEMS ${dependencies}) - find_package(${Dependency} REQUIRED) -endforeach() -find_package(fluent_bit REQUIRED) -set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH}) -find_package(Sanitizers) - -# C plugins -list(APPEND c_plugins "in_ros2") - -foreach(c_plugin ${c_plugins}) - set(LIBRARY_NAME "flb-${c_plugin}") - add_library(${LIBRARY_NAME} SHARED "src/c/${c_plugin}.c") - ament_target_dependencies(${LIBRARY_NAME} ${dependencies}) - add_sanitizers(${LIBRARY_NAME}) - set_target_properties(${LIBRARY_NAME} PROPERTIES PREFIX "") - - list(APPEND c_libs ${LIBRARY_NAME}) -endforeach() - -# Go Plugins -if(EXISTS "/usr/bin/go") - list(APPEND go_plugins "out_minio") - list(APPEND go_plugins "out_files_metrics") - - foreach(go_plugin ${go_plugins}) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/go/${go_plugin}/) - endforeach() -endif() - -include_directories( - include -) - -install(DIRECTORY include/ - DESTINATION include/ -) - -install(TARGETS ${c_libs} - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin -) - -ament_export_include_directories(include) -ament_export_library_names(${c_libs} ${go_libs}) -ament_export_dependencies(${dependencies}) -ament_package() diff --git a/fluent_bit_plugins/COLCON_IGNORE b/fluent_bit_plugins/COLCON_IGNORE deleted file mode 100644 index 045fb9b16..000000000 --- a/fluent_bit_plugins/COLCON_IGNORE +++ /dev/null @@ -1,3 +0,0 @@ -Ignored on the jazzy line pending the DC 2.0 embedded-Fluent-Bit demolition (ADR-0001: -replace the embedded Fluent Bit engine with an external Vector shipper). See the dc-2.0 -epic (#241). Do not delete this package; it is dropped, not removed. diff --git a/fluent_bit_plugins/include/fluent_bit_plugins/in_ros2.h b/fluent_bit_plugins/include/fluent_bit_plugins/in_ros2.h deleted file mode 100644 index ed0c15de9..000000000 --- a/fluent_bit_plugins/include/fluent_bit_plugins/in_ros2.h +++ /dev/null @@ -1,67 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2022 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_ROS2_H -#define FLB_IN_ROS2_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dc_interfaces/msg/string_stamped.h" -#include "rcl/error_handling.h" -#include "rclc/executor.h" -#include "rclc/rclc.h" - -struct rclc_subscriber -{ - rcl_subscription_t data_subscription; - const rosidl_message_type_support_t* data_type_support; - rcl_subscription_options_t subscription_options; - char* topic_name; - dc_interfaces__msg__StringStamped data_msg; - struct mk_list _head; -}; - -/* ROS2 Input configuration & context */ -struct flb_ros2 -{ - struct mk_list* topics_list; /* Topic names as a list */ - struct mk_list topic_subs; /* topics subscribers */ - flb_sds_t node_name; /* Name of the node */ - char* topics; /* Topic names as a string */ - int spin_time; /* Time to wait, in ms */ - int buf_len; /* read buffer length */ - char* buf; /* read buffer */ - struct flb_pack_state pack_state; - struct flb_input_instance* ins; -}; - -extern struct flb_input_plugin in_ros2_plugin; - -#endif diff --git a/fluent_bit_plugins/package.xml b/fluent_bit_plugins/package.xml deleted file mode 100644 index 384446120..000000000 --- a/fluent_bit_plugins/package.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - fluent_bit_plugins - 0.1.0 - A vendor package for An End to End Observability Pipeline - David Bensoussan - MPL-2.0 - - golang-go - - dc_interfaces - fluent_bit_vendor - rcl - rclc - - - ament_cmake - - diff --git a/fluent_bit_plugins/src/c/in_ros2.c b/fluent_bit_plugins/src/c/in_ros2.c deleted file mode 100644 index cf8b7dd6e..000000000 --- a/fluent_bit_plugins/src/c/in_ros2.c +++ /dev/null @@ -1,431 +0,0 @@ -#include "fluent_bit_plugins/in_ros2.h" - -// Ideally we pass pointers and not use global variables -rclc_executor_t executor; -rcl_node_t node; -rcl_init_options_t init_options; -rcl_context_t context; -rcl_allocator_t allocator; -rcl_node_options_t node_ops; -void* glob_ctx; -static volatile bool new_data_flag = false; - -static int config_destroy(struct flb_ros2* ctx) -{ - // Destroy topic_subs - struct mk_list* tmp; - struct mk_list* head; - struct rclc_subscriber* subscriber; - mk_list_foreach_safe(head, tmp, &ctx->topic_subs) - { - subscriber = mk_list_entry(head, struct rclc_subscriber, _head); - mk_list_del(&subscriber->_head); - flb_free(subscriber); - } - - // Destroy topics_list - struct mk_list* topics_list = (struct mk_list*)ctx->topics_list; - struct flb_slist_entry* entry; - - if (topics_list != NULL) - { - mk_list_foreach_safe(head, tmp, topics_list) - { - entry = mk_list_entry(head, struct flb_slist_entry, _head); - if (entry != NULL) - { - mk_list_del(&entry->_head); - flb_free(entry); - } - } - } - - flb_free(topics_list); - - // Free ctx - flb_free(ctx); - return 0; -} - -static int set_timestamp(msgpack_packer* mp_pck, const dc_interfaces__msg__StringStamped* msg) -{ - struct flb_time msg_time = { .tm.tv_sec = msg->header.stamp.sec, .tm.tv_nsec = msg->header.stamp.nanosec }; - int ret = flb_time_append_to_msgpack(&msg_time, mp_pck, 0); - - return ret; -} - -static inline int process_pack(msgpack_packer* mp_pck, char* data, size_t data_size, - dc_interfaces__msg__StringStamped* msg) -{ - size_t off = 0; - msgpack_unpacked result; - msgpack_object entry; - - /* Queue the data with time field */ - msgpack_unpacked_init(&result); - - while (msgpack_unpack_next(&result, data, data_size, &off) == MSGPACK_UNPACK_SUCCESS) - { - entry = result.data; - - if (entry.type == MSGPACK_OBJECT_MAP) - { - flb_debug("MSGPACK_OBJECT_MAP"); - msgpack_pack_array(mp_pck, 2); - set_timestamp(mp_pck, msg); - msgpack_pack_object(mp_pck, entry); - } - else if (entry.type == MSGPACK_OBJECT_ARRAY) - { - flb_debug("MSGPACK_OBJECT_ARRAY"); - msgpack_pack_object(mp_pck, entry); - } - else - { - /* - * Upon exception, acknowledge the user about the problem but continue - * working, do not discard valid JSON entries. - */ - flb_error("invalid record found, it's not a JSON map or array"); - msgpack_unpacked_destroy(&result); - return -1; - } - } - - msgpack_unpacked_destroy(&result); - return 0; -} -void data_callback(const void* msgin) -{ - dc_interfaces__msg__StringStamped* msg = (dc_interfaces__msg__StringStamped*)msgin; - if (msg == NULL) - { - flb_warn("Callback: msg NULL\n"); - } - else - { - flb_debug("Callback: I heard: ts=%d.%d data=%s \n", msg->header.stamp.sec, msg->header.stamp.nanosec, - msg->data.data); - struct flb_ros2* ctx = glob_ctx; - - msgpack_sbuffer mp_sbuf; - msgpack_unpacked result; - msgpack_packer mp_pck; - int root_type; - - /* Queue the data with time field */ - msgpack_unpacked_init(&result); - - /* Initialize local msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - size_t pack_size; - char* pack; - ctx->buf = msg->data.data; - ctx->buf_len = msg->data.capacity; - flb_debug("buf_len: %d, buf: '%s'", ctx->buf_len, ctx->buf); - - int ret = flb_pack_json(ctx->buf, ctx->buf_len, &pack, &pack_size, &root_type, NULL); - if (ret == FLB_ERR_JSON_PART) - { - flb_warn("Data incomplete, waiting for more..."); - msgpack_sbuffer_destroy(&mp_sbuf); - } - else if (ret == FLB_ERR_JSON_INVAL) - { - flb_warn("Invalid JSON message, skipping"); - msgpack_sbuffer_destroy(&mp_sbuf); - } - else if (ret == 0) - { - flb_debug("Data complete"); - /* Process valid packaged records */ - process_pack(&mp_pck, pack, pack_size, msg); - - flb_pack_state_reset(&ctx->pack_state); - flb_pack_state_init(&ctx->pack_state); - flb_free(pack); - - flb_input_log_append(ctx->ins, NULL, 0, mp_sbuf.data, mp_sbuf.size); - msgpack_sbuffer_destroy(&mp_sbuf); - } - } -} - -static int ros2_rclc_init() -{ - /* Initialize rclc */ - allocator = rcl_get_default_allocator(); - node_ops = rcl_node_get_default_options(); - - /* Define ROS context */ - context = rcl_get_zero_initialized_context(); - - /* Create init_options */ - rcl_ret_t rc = rcl_init_options_init(&init_options, allocator); - if (rc != RCL_RET_OK) - { - flb_error("Error rcl_init_options_init: %d.\n", rc); - return -1; - } - - /* Create context */ - rc = rcl_init(0, NULL, &init_options, &context); - if (rc != RCL_RET_OK) - { - flb_error("Error in rcl_init.\n"); - return -1; - } - return 0; -} - -static int ros2_node_init(struct flb_ros2* ctx) -{ - /* Create node */ - node = rcl_get_zero_initialized_node(); - const char* node_name = ctx->node_name; - rcl_ret_t rc = rcl_node_init(&node, node_name, "/", &context, &node_ops); - if (rc != RCL_RET_OK) - { - flb_error("Error in rclc_node_init\n"); - return -1; - } - else - { - flb_info("Started node %s", node_name); - } - return 0; -} - -static int ros2_subscribers_init(struct flb_ros2* ctx) -{ - struct mk_list* head; - struct rclc_subscriber* subscriber; - struct flb_slist_entry* topic = NULL; - - int len = mk_list_size(ctx->topics_list); - if (!ctx->topics_list || len == 0) - { - flb_error("No 'topics' options has been specified."); - return 0; - } - mk_list_init(&ctx->topic_subs); - - mk_list_foreach(head, ctx->topics_list) - { - topic = mk_list_entry(head, struct flb_slist_entry, _head); - subscriber = flb_malloc(sizeof(struct rclc_subscriber)); - if (!subscriber) - { - flb_errno(); - return -1; - } - subscriber->data_subscription = rcl_get_zero_initialized_subscription(); - subscriber->data_type_support = ROSIDL_GET_MSG_TYPE_SUPPORT(dc_interfaces, msg, StringStamped); - subscriber->subscription_options = rcl_subscription_get_default_options(); - - rcl_ret_t rc = rcl_subscription_init(&(subscriber->data_subscription), &node, subscriber->data_type_support, - topic->str, &(subscriber->subscription_options)); - if (rc != RCL_RET_OK) - { - flb_error("Failed to create subscriber %s.\n", topic->str); - return -1; - } - else - { - flb_info("Created subscriber %s", topic->str); - } - if (false == dc_interfaces__msg__StringStamped__init(&(subscriber->data_msg))) - { - flb_error("Failed to init msg.\n"); - return -1; - } - - mk_list_add(&subscriber->_head, &(ctx->topic_subs)); - } - - return 0; -} - -static int ros2_executor_init(struct flb_ros2* ctx) -{ - /* Executor */ - int len = mk_list_size(ctx->topics_list); - rclc_executor_init(&executor, &context, len, &allocator); - - /* Add subscriptions to executor */ - struct mk_list* head; - struct mk_list* tmp; - struct rclc_subscriber* an_item; - rcl_ret_t rc; - - mk_list_foreach_safe(head, tmp, &ctx->topic_subs) - { - an_item = mk_list_entry(head, struct rclc_subscriber, _head); - rc = rclc_executor_add_subscription(&executor, &an_item->data_subscription, &(an_item->data_msg), &data_callback, - ON_NEW_DATA); - if (rc != RCL_RET_OK) - { - flb_error("Error in rclc_executor_add_subscription.\n"); - } - } - rc = rclc_executor_prepare(&executor); - if (rc != RCL_RET_OK) - { - flb_error("Error in rclc_executor_prepare.\n"); - } - - return 0; -} - -/* cb_collect callback */ -static int in_ros2_collect(struct flb_input_instance* ins, struct flb_config* config, void* in_context) -{ - struct flb_ros2* ctx = in_context; - rclc_executor_spin_some(&executor, RCL_MS_TO_NS(ctx->spin_time)); - return 0; -} - -/* Initialize plugin */ -static int in_ros2_init(struct flb_input_instance* in, struct flb_config* config, void* data) -{ - int ret = -1; - struct flb_ros2* ctx; - - /* Allocate space for the configuration */ - ctx = flb_calloc(1, sizeof(struct flb_ros2)); - if (!ctx) - { - return -1; - } - ctx->ins = in; - - const char* node_name; - - node_name = (char*)flb_input_get_property("node_name", in); - if (node_name == NULL) - { - ctx->node_name = "fluentbit_rclc"; - } - else - { - ctx->node_name = (char*)node_name; - } - const char* spin_time; - spin_time = flb_input_get_property("spin_time", in); - if (spin_time == NULL) - { - ctx->spin_time = 100; - } - else - { - ctx->spin_time = atoi(spin_time); - } - - ctx->topics_list = flb_malloc(sizeof(struct mk_list)); - if (!ctx->topics_list) - { - flb_errno(); - return -1; - } - mk_list_init(ctx->topics_list); - - const char* topics; - topics = flb_input_get_property("topics", in); - ctx->topics = (char*)topics; - int max_split = 0; - ret = flb_slist_split_tokens(ctx->topics_list, ctx->topics, max_split); - - if (ret == -1) - { - config_destroy(ctx); - return -1; - } - - ret = ros2_rclc_init(); - if (ret == -1) - { - config_destroy(ctx); - return -1; - } - - ret = ros2_node_init(ctx); - if (ret == -1) - { - config_destroy(ctx); - return -1; - } - - ret = ros2_subscribers_init(ctx); - if (ret == -1) - { - config_destroy(ctx); - return -1; - } - - ret = ros2_executor_init(ctx); - if (ret == -1) - { - config_destroy(ctx); - return -1; - } - - glob_ctx = ctx; - /* Always initialize built-in JSON pack state */ - flb_pack_state_init(&ctx->pack_state); - /* Load fluentbit context config */ - flb_input_set_context(in, ctx); - /* Fluentbit collect */ - ret = flb_input_set_collector_time(in, in_ros2_collect, 0, 1000000, config); - if (ret < 0) - { - flb_error("Could not set collector for ros2 input plugin"); - flb_free(ctx->topics_list); - return -1; - } - - return 0; -} - -static int in_ros2_exit(void* data, struct flb_config* config) -{ - /* Clean up */ - rcl_ret_t rc = RCL_RET_OK; - struct rclc_subscriber* an_item; - struct mk_list* head; - struct mk_list* tmp; - struct flb_ros2* ctx = data; - - mk_list_foreach_safe(head, tmp, &ctx->topic_subs) - { - an_item = mk_list_entry(head, struct rclc_subscriber, _head); - rc = rcl_subscription_fini(&an_item->data_subscription, &node); - } - rc += rcl_node_fini(&node); - rc += rcl_init_options_fini(&init_options); - mk_list_foreach_safe(head, tmp, &ctx->topic_subs) - { - an_item = mk_list_entry(head, struct rclc_subscriber, _head); - dc_interfaces__msg__StringStamped__fini(&(an_item->data_msg)); - } - rc += rclc_executor_fini(&executor); - - if (rc != RCL_RET_OK) - { - flb_error("Error while cleaning up!\n"); - return -1; - } - - config_destroy(ctx); - return 0; -} - -struct flb_input_plugin in_ros2_plugin = { .name = "ros2", - .description = "ROS2 Input", - .cb_init = in_ros2_init, - .cb_pre_run = NULL, - .cb_collect = in_ros2_collect, - .cb_flush_buf = NULL, - .cb_exit = in_ros2_exit }; diff --git a/fluent_bit_plugins/src/go/out_files_metrics/CMakeLists.txt b/fluent_bit_plugins/src/go/out_files_metrics/CMakeLists.txt deleted file mode 100644 index c22338385..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -cmake_minimum_required(VERSION 3.0) -project(out_files_metrics) - -set(GO_COMPILER go) -set(FILES_METRICS_LIBRARY out_files_metrics.so) -set(FILES_METRICS_INCLUDE out_files_metrics.h) - -add_custom_target(plugin_out_files_metrics_install ALL - COMMAND ${GO_COMPILER} get - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target(plugin_out_files_metrics ALL - COMMAND ${GO_COMPILER} build -buildmode=c-shared -buildvcs=false -o ${FILES_METRICS_LIBRARY} ${CMAKE_CURRENT_SOURCE_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -install( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/${FILES_METRICS_LIBRARY} - FILES ${CMAKE_CURRENT_SOURCE_DIR}/${FILES_METRICS_INCLUDE} - DESTINATION lib -) diff --git a/fluent_bit_plugins/src/go/out_files_metrics/db_pgsql.go b/fluent_bit_plugins/src/go/out_files_metrics/db_pgsql.go deleted file mode 100644 index 08d9219a7..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/db_pgsql.go +++ /dev/null @@ -1,95 +0,0 @@ -package main - -import ( - "C" - "database/sql" - "fmt" - "unsafe" - - "github.com/fluent/fluent-bit-go/output" - _ "github.com/lib/pq" -) - -type PGSQLConfig struct { - host string - port string - user string - password string - database string - table string - time_key string - ssl string -} - -func PGSQLInit(plugin unsafe.Pointer) error { - plugin_conf.db.postgres = true - pgsql_config = PGSQLConfig{ - host: output.FLBPluginConfigKey(plugin, "pgsql_host"), - port: output.FLBPluginConfigKey(plugin, "pgsql_port"), - user: output.FLBPluginConfigKey(plugin, "pgsql_user"), - password: output.FLBPluginConfigKey(plugin, "pgsql_password"), - database: output.FLBPluginConfigKey(plugin, "pgsql_database"), - table: output.FLBPluginConfigKey(plugin, "pgsql_table"), - time_key: output.FLBPluginConfigKey(plugin, "pgsql_timestamp_key"), - ssl: output.FLBPluginConfigKey(plugin, "pgsql_use_ssl"), - } - - connStr := fmt.Sprintf("postgres://%s:%s@%s:%s?sslmode=%s", pgsql_config.user, pgsql_config.password, pgsql_config.host, pgsql_config.port, pgsql_config.ssl) - fmt.Printf("[flb-files-metric] PGSQL connstr: '%s'\n", connStr) - var err error - db, _ = sql.Open("postgres", connStr) - err = db.Ping() - if err != nil { - fmt.Println("[flb-files-metric] Could not ping Postgres") - return err - } - var exists bool - fmt.Printf("[flb-files-metric] Trying to find database %s...\n", pgsql_config.database) - err = db.QueryRow(fmt.Sprintf("SELECT 1 from pg_database WHERE datname='%s'", pgsql_config.database)).Scan(&exists) - - if err != nil && err != sql.ErrNoRows { - fmt.Println("[flb-files-metric] Error checking if database exists") - } else { - if exists { - fmt.Println("[flb-files-metric] Database exists") - } else { - _, err = db.Exec("CREATE DATABASE " + pgsql_config.database) - if err != nil { - return err - } - fmt.Printf("[flb-files-metric] Created database %s\n", pgsql_config.database) - } - } - - _, table_check := db.Query("select * from " + pgsql_config.table + ";") - - if table_check == nil { - fmt.Println("[flb-files-metric] Table exists") - } else { - fmt.Println("[flb-files-metric] Table not found, creating...") - query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s( - id SERIAL PRIMARY KEY, - timestamp TIMESTAMP NOT NULL, - robot_name TEXT NOT NULL, - robot_id TEXT NOT NULL, - group_name TEXT NOT NULL, - duration FLOAT8, - local_path TEXT NOT NULL, - remote_path TEXT NOT NULL, - uploaded BOOLEAN NOT NULL, - on_filesystem BOOLEAN NOT NULL, - deleted BOOLEAN NOT NULL, - storage_type TEXT NOT NULL, - content_type TEXT NOT NULL, - size INTEGER NOT NULL, - updated_at TIMESTAMP NOT NULL)`, pgsql_config.table) - _, err := db.ExecContext(ctx, query) - if err != nil { - fmt.Printf("[flb-files-metric] Error %s when creating table", err) - return err - } else { - fmt.Printf("[flb-files-metric] Created table %s\n", pgsql_config.table) - } - } - return err -} diff --git a/fluent_bit_plugins/src/go/out_files_metrics/go.mod b/fluent_bit_plugins/src/go/out_files_metrics/go.mod deleted file mode 100644 index 2c0ccc1a3..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -module out_files_metrics - -go 1.18 - -require ( - github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d - github.com/lib/pq v1.10.9 - github.com/minio/minio-go/v7 v7.0.56 -) - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.2 // indirect - github.com/ugorji/go/codec v1.1.7 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect -) diff --git a/fluent_bit_plugins/src/go/out_files_metrics/go.sum b/fluent_bit_plugins/src/go/out_files_metrics/go.sum deleted file mode 100644 index e0eb010df..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/go.sum +++ /dev/null @@ -1,55 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d h1:b4a4JIzP5VT7l64NHnXF1nP95zRRN7wVMsD5D1jJiY0= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d/go.mod h1:L92h+dgwElEyUuShEwjbiHjseW410WIcNz+Bjutc8YQ= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY= -github.com/minio/minio-go/v7 v7.0.56/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/fluent_bit_plugins/src/go/out_files_metrics/misc.go b/fluent_bit_plugins/src/go/out_files_metrics/misc.go deleted file mode 100644 index f9713389a..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/misc.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "fmt" -) - -func contains(elems []int, v int) bool { - for _, s := range elems { - if v == s { - return true - } - } - return false -} - -func containsStr(elems []string, v string) bool { - for _, s := range elems { - if v == s { - return true - } - } - return false -} - -type M map[string]interface{} - -func NestedMapLookup(m map[interface{}]interface{}, ks ...string) (rval interface{}, err error) { - var ok bool - - if len(ks) == 0 { // degenerate input - return nil, fmt.Errorf("NestedMapLookup needs at least one key") - } - if rval, ok = m[ks[0]]; !ok { - return nil, fmt.Errorf("key not found; remaining keys: %v", ks) - } else if len(ks) == 1 { // we've reached the final key - return rval, nil - } else if m, ok = rval.(map[interface{}]interface{}); !ok { - return nil, fmt.Errorf("malformed structure at %#v", rval) - } else { // 1+ more keys - return NestedMapLookup(m, ks[1:]...) - } -} diff --git a/fluent_bit_plugins/src/go/out_files_metrics/out_files_metrics.go b/fluent_bit_plugins/src/go/out_files_metrics/out_files_metrics.go deleted file mode 100644 index ed3448665..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/out_files_metrics.go +++ /dev/null @@ -1,484 +0,0 @@ -package main - -import ( - "C" - "context" - "database/sql" - "fmt" - "net/http" - "os" - "os/exec" - "strconv" - "strings" - "time" - "unsafe" - - "github.com/fluent/fluent-bit-go/output" - _ "github.com/lib/pq" - // "github.com/minio/minio-go/v7/pkg/credentials" -) -import ( - "errors" - - "github.com/minio/minio-go/v7" -) - -type StorageConfig struct { - minio bool - s3 bool -} - -type DBConfig struct { - postgres bool -} - -type PluginConfig struct { - storage StorageConfig - db DBConfig -} - -// General config -var storage = make([]string, 0) -var plugin_conf PluginConfig - -/* -{ - "src": ["p1", "p2"], - "dest": [{"s3": "rp1", "minio": "foo/rp1"}, {"s3": "rp2"}] - "uploaded": [true, true] -} -*/ - -// var uploaded = make(map[string]bool) - -var db *sql.DB -var db_type string -var ctx = context.Background() -var pgsql_config PGSQLConfig -var delete_when_sent bool - -func get_file_metadata(path string) (string, int64) { - file, err := os.Open(path) - - defer file.Close() - - if err != nil { - return "", 0 - } - // Get the file content - buf := make([]byte, 512) - _, err = file.Read(buf) - - if err != nil { - return "", 0 - } - - defer file.Close() - - fileInfo, err := file.Stat() - if err != nil { - fmt.Println(err) - return "", 0 - } - - contentType := http.DetectContentType(buf) - - return contentType, fileInfo.Size() -} - -func getDurationVideo(path string) float64 { - // Execute the "ffprobe" command to get the size of a file - cmd := exec.Command("sh", "-c", fmt.Sprintf("/usr/bin/ffprobe -i %s -show_entries format=duration -v quiet -of csv=\"p=0\"", path)) - - stdout, err := cmd.Output() - - if err != nil { - fmt.Println(err.Error()) - return -1.0 - } - - // Print the output - result := string(stdout) - result = strings.TrimSuffix(result, "\n") - - f_result, err := strconv.ParseFloat(result, 64) - if err != nil { - fmt.Printf("Could not get the duration when parsing") - result := -1.0 - return result - } - return f_result -} - -//export FLBPluginRegister -func FLBPluginRegister(ctx unsafe.Pointer) int { - return output.FLBPluginRegister(ctx, "files_metrics", "Files metrics GO!") -} - -func getDateFromPath(path string) string { - // Split the string by '/' character - fmt.Printf("[flb-files-metrics] path=%s\n", path) - parts := strings.Split(path, "/") - - // Get the last element in the slice - lastIndex := len(parts) - 1 - dateTimeString := parts[lastIndex] - fmt.Printf("[flb-files-metrics] dateTimeString=%s\n", dateTimeString) - - // Find the index of the dot character - dotIndex := strings.Index(dateTimeString, ".") - fmt.Printf("[flb-files-metrics] dotIndex=%d\n", dotIndex) - - // Trim the characters starting from the dot character - dateTimeString = strings.TrimSuffix(dateTimeString[:dotIndex], ".") - fmt.Printf("[flb-files-metrics] dateTimeString=%s\n", dateTimeString) - - var layout string = "2006-01-02T15-04-05" - - date, err := time.Parse(layout, dateTimeString) - if err != nil { - fmt.Println(err) - } - return date.UTC().Format(time.RFC3339Nano) -} - -// (fluentbit will call this) -// plugin (context) pointer to fluentbit context (state/ c code) -// -//export FLBPluginInit -func FLBPluginInit(plugin unsafe.Pointer) int { - fmt.Printf("[flb-files-metrics] Init plugin...\n") - storage = strings.Fields(output.FLBPluginConfigKey(plugin, "file_storage")) - delete_when_sent_str := output.FLBPluginConfigKey(plugin, "delete_when_sent") - db_type = output.FLBPluginConfigKey(plugin, "db_type") - var err error - delete_when_sent, err = strconv.ParseBool(delete_when_sent_str) - if err != nil { - return output.FLB_RETRY - } - - // Storage - // Minio - fmt.Printf("[flb-files-metrics] Initializing storages...: %s\n", storage) - for _, val := range storage { - if strings.ToLower(val) == "minio" { - minio_config = MinioInitConf(plugin) - if minio_config.endpoint == "" { - fmt.Printf("[flb-files-metrics] Failure when initializing Storage MinIO\n") - return output.FLB_RETRY - } - } else if strings.ToLower(val) == "s3" { - s3_config = S3InitConf(plugin) - if s3_config.endpoint == "" { - fmt.Printf("[flb-files-metrics] Failure when initializing Storage S3\n") - return output.FLB_RETRY - } - } - } - fmt.Printf("[flb-files-metrics] Initialized storages\n") - - // Database - // PGSQL - if strings.ToLower(db_type) == "pgsql" { - init_err := PGSQLInit(plugin) - if init_err != nil { - fmt.Printf("[flb-files-metrics] Cannot initialize PGSQL\n") - return output.FLB_RETRY - } else { - fmt.Printf("[flb-files-metrics] Initialized PGSQL\n") - } - } else { - fmt.Printf("[flb-files-metrics] No database setup. Stopping.\n") - return output.FLB_RETRY - } - - fmt.Printf("[flb-files-metrics] Initialized plugin\n") - - return output.FLB_OK -} - -func getPath(record map[interface{}]interface{}, dot_separated_path string) string { - var key_lookup string = "" - var path string = "" - - if record["nested"] == true && record["flattened"] == false { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == true && record["flattened"] == true { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) + "/" + strings.Replace(dot_separated_path, ".", "/", -1) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == true && record["flattened"] == false { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) + "/" + strings.Replace(dot_separated_path, ".", "/", -1) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == false && record["flattened"] == false { - local_path_val, err := NestedMapLookup(record, strings.Split(dot_separated_path, ".")...) - if err == nil { - path = fmt.Sprintf("%s", local_path_val) - } - } - - return path -} - -type FileToAnalyze struct { - RobotName string - GroupName string - LocalPath string - UploadPath string - Uploaded []bool - UploadTarget []string - // UploadCurrent []string - ContentType string - Size int64 -} - -func isFileUploaded(storage string, remote_path string) bool { - if plugin_conf.storage.minio && storage == "minio" { - _, err_found := minio_client.StatObject(ctx, minio_config.bucket, remote_path, minio.StatObjectOptions{}) - if err_found == nil { - return true - } else { - return false - } - } - - return true -} - -func findFileByLocalPath(fileList []FileToAnalyze, searchPath string) (bool, int) { - for index, file := range fileList { - if file.LocalPath == searchPath { - return true, index - } - } - return false, -1 -} - -//export FLBPluginFlush -func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { - var ret int - var ts interface{} - var record map[interface{}]interface{} - var files []FileToAnalyze - var split_src_fields = make([]string, 0) - var split_upload_fields = make([]string, 0) - - // Create Fluent Bit decoder - dec := output.NewDecoder(data, int(length)) - retry := false - - // Connect to remote services - if plugin_conf.storage.minio { - init_err := MinioInit() - if init_err != nil { - fmt.Printf("[flb-files-metrics] Cannot initialize Minio\n") - return output.FLB_RETRY - } - fmt.Printf("[flb-files-metrics] Initialized Minio\n") - } - if plugin_conf.storage.s3 { - init_err := S3Init() - if init_err != nil { - fmt.Printf("[flb-files-metrics] Cannot initialize S3\n") - return output.FLB_RETRY - } - fmt.Printf("[flb-files-metrics] Initialized S3\n") - } - - for { - // Extract Record - ret, ts, record = output.GetRecord(dec) - if ret != 0 || ts == nil { - fmt.Printf("[flb-files-metrics] Could not get records, ts = %d, ret = %d\n", ts, ret) - break - } - if ret != 0 { - break - } - fmt.Printf("[flb-files-metrics] Got records %s!\n", record) - - var current_group string - - // Loop through each storage type - for _, storage_type := range storage { - // Associate functions to store paths depending on storage type - if strings.ToLower(storage_type) == "minio" { - split_src_fields = minio_split_src_fields - split_upload_fields = minio_split_upload_fields - } else if strings.ToLower(storage_type) == "s3" { - split_src_fields = s3_split_src_fields - split_upload_fields = s3_split_upload_fields - } - - // Ensure length of src and upload is the same. If not, stop the plugin - // The configuration is wrong - if len(split_upload_fields) != len(split_src_fields) { - // Throw an error - err := fmt.Errorf("Source and destination fields must be of same length") - panic(err) - } - - // Iterate though each src_fields path - for split_src_field_i, split_src_field_v := range split_src_fields { - var uploaded bool - // Each field is associated with a group equal to its measurement name - // We find all fields for the same measurement name. - fmt.Printf("[flb-files-metrics] current_group=%s\n", current_group) - on_filesystem := true - var local_path string = "" - // Set group if it is not yet done - var file_to_analyze FileToAnalyze - // If first path received, set the group - if len(current_group) == 0 { - // Get path from the field name and record received - local_path = getPath(record, split_src_field_v) - fmt.Printf("[flb-files-metrics] Found %s, split_src_field_v %s\n", local_path, split_src_field_v) - - // Add to group if the file exists - if len(local_path) != 0 { - if _, err := os.Stat(local_path); errors.Is(err, os.ErrNotExist) { - // File does not exist - fmt.Printf("[flb-files-metrics] File %s does not exist\n", local_path) - on_filesystem = false - } else { - current_group = fmt.Sprintf("%s", record["name"]) - uploaded = isFileUploaded(storage_type, getPath(record, split_upload_fields[split_src_field_i])) - var content_type string - var size int64 - if !uploaded { - retry = true - } else { - content_type, size = get_file_metadata(fmt.Sprintf("%s", local_path)) - } - file_to_analyze = FileToAnalyze{ - RobotName: string(record["robot_name"].([]uint8)), - GroupName: current_group, - LocalPath: local_path, - UploadPath: getPath(record, split_upload_fields[split_src_field_i]), - UploadTarget: []string{storage_type}, - Uploaded: []bool{uploaded}, - ContentType: content_type, - Size: size, - } - files = append(files, file_to_analyze) - fmt.Printf("[flb-files-metrics] File %s exists, setting group to %s\n", local_path, current_group) - } - } - } else if current_group == fmt.Sprintf("%s", record["name"]) { - fmt.Printf("[flb-files-metrics] split_src_field_i match index %d: current_group=%s, split_src_field_v=%s\n", split_src_field_i, current_group, split_src_field_v) - local_path = getPath(record, split_src_field_v) - - // Add to group if the file exists - if len(local_path) != 0 { - if _, err := os.Stat(local_path); errors.Is(err, os.ErrNotExist) { - // File does not exist - on_filesystem = false - fmt.Printf("[flb-files-metrics] File %s does not exist\n", local_path) - } else { - // File was maybe added before, when adding it for another storage type - found, index := findFileByLocalPath(files, local_path) - uploaded = isFileUploaded(storage_type, getPath(record, split_upload_fields[split_src_field_i])) - var content_type string - var size int64 - if found { - files[index].UploadTarget = append(files[index].UploadTarget, storage_type) - if !uploaded { - retry = true - } else { - content_type, size = get_file_metadata(fmt.Sprintf("%s", local_path)) - } - files[index].Uploaded = append(files[index].Uploaded, uploaded) - } else { - content_type, size = get_file_metadata(fmt.Sprintf("%s", local_path)) - file_to_analyze = FileToAnalyze{ - RobotName: string(record["robot_name"].([]uint8)), - GroupName: current_group, - LocalPath: local_path, - UploadPath: getPath(record, split_upload_fields[split_src_field_i]), - UploadTarget: []string{storage_type}, - Uploaded: []bool{uploaded}, - ContentType: content_type, - Size: size, - } - files = append(files, file_to_analyze) - } - } - } - } - if len(local_path) != 0 { - fmt.Printf("[flb-files-metrics] Files %v\n", files) - _, index := findFileByLocalPath(files, local_path) - if index != -1 { - file := files[index] - optional_fields := "" - optional_values := "" - if file.ContentType == "video/mp4" || file.ContentType == "application/octet-stream" { - optional_fields += ", duration" - optional_values += fmt.Sprintf(", '%f'", getDurationVideo(fmt.Sprintf("%s", file.LocalPath))) - } - timestamp := getDateFromPath(fmt.Sprintf("%s", file.LocalPath)) - updated_at := time.Now().UTC().Format(time.RFC3339Nano) - robot_id := record["id"] - prefix_path := "" - if plugin_conf.storage.minio && storage_type == "minio" { - prefix_path = "minio://" + minio_config.bucket + "/" - } else if plugin_conf.storage.minio && storage_type == "s3" { - prefix_path = "s3://" + s3_config.bucket + "/" - } - sqlStatement := fmt.Sprintf(`INSERT INTO %s(timestamp, robot_id, robot_name, group_name, local_path, remote_path, uploaded, on_filesystem, deleted, storage_type, content_type, size, updated_at %s) - VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %t, %t, false, '%s', '%s', %d, '%s'%s);`, pgsql_config.table, optional_fields, timestamp, robot_id, file.RobotName, file.GroupName, file.LocalPath, prefix_path+file.UploadPath, uploaded, on_filesystem, storage_type, file.ContentType, file.Size, updated_at, optional_values) - fmt.Printf("[flb-files-metrics] Statement: %s\n", sqlStatement) - _, err := db.Exec(sqlStatement) - if err != nil { - retry = true - fmt.Printf("[flb-files-metrics] Could not insert in DB: %s\n", err) - } else { - fmt.Printf("[flb-files-metrics] Inserted in DB\n") - } - - // If uploaded everywhere - if !retry { - fmt.Printf("[flb-files-metrics] Uploaded everywhere %s\n", local_path) - if delete_when_sent { - fmt.Printf("[flb-files-metrics] Deleting ...\n") - os.Remove(fmt.Sprintf("%s", local_path)) - // Ignore errors of os.Remove. - // We should not retry because it has already been sent. It is possible the file was sent and application stopped and when starting again, want to send again - // So we ignore it. - sqlStatement := fmt.Sprintf(`UPDATE %s - SET deleted = true, - updated_at = '%s' - WHERE local_path = '%s';`, pgsql_config.table, time.Now().UTC().Format(time.RFC3339Nano), local_path) - fmt.Printf("[flb-files-metrics] Statement: %s\n", sqlStatement) - _, err = db.Exec(sqlStatement) - if err != nil { - fmt.Printf("[flb-files-metrics] Could not update DB\n") - } else { - fmt.Printf("[flb-files-metrics] Updated DB\n") - } - } - } else { - retry = true - fmt.Printf("[flb-files-metrics] Not uploaded everywhere %s\n", local_path) - } - } - } - } - } - fmt.Printf("[flb-files-metrics] Files %v\n", files) - - } - - if retry == true { - return output.FLB_RETRY - } - return output.FLB_OK -} - -//export FLBPluginExit -func FLBPluginExit() int { - return output.FLB_OK -} - -func main() { -} diff --git a/fluent_bit_plugins/src/go/out_files_metrics/storage_minio.go b/fluent_bit_plugins/src/go/out_files_metrics/storage_minio.go deleted file mode 100644 index ff5729449..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/storage_minio.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "C" - "fmt" - "strconv" - "strings" - "unsafe" - - "github.com/fluent/fluent-bit-go/output" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -type MinioConfig struct { - endpoint string - access_key_id string - secret_access_key string - use_ssl bool - bucket string - src_fields []string - upload_fields []string -} - -// Minio -var minio_config MinioConfig -var minio_client *minio.Client -var minio_endpoint string -var minio_access_key_id string -var minio_secret_access_key string -var minio_use_ssl string -var minio_create_bucket string -var minio_bucket string -var minio_upload_fields string -var minio_src_fields string -var minio_split_upload_fields = make([]string, 0) -var minio_split_src_fields = make([]string, 0) - -func MinioInit() error { - // Initialize minio client object. - var err error - minio_client, err = minio.New(minio_config.endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(minio_config.access_key_id, minio_config.secret_access_key, ""), - Secure: minio_config.use_ssl, - }) - - return err -} - -func MinioInitConf(plugin unsafe.Pointer) MinioConfig { - plugin_conf.storage.minio = true - fmt.Printf("[flb-files-metric] Initializing Minio...\n") - minio_use_ssl = output.FLBPluginConfigKey(plugin, "minio_use_ssl") - use_ssl_bool, err := strconv.ParseBool(minio_use_ssl) - if err != nil { - fmt.Printf("[flb-files-metric] Could not parse use_ssl\n") - return MinioConfig{} - } - - minio_src_fields = output.FLBPluginConfigKey(plugin, "minio_src_fields") - minio_split_src_fields = strings.Fields(minio_src_fields) - minio_upload_fields = output.FLBPluginConfigKey(plugin, "minio_upload_fields") - minio_split_upload_fields = strings.Fields(minio_upload_fields) - - minio_config = MinioConfig{ - endpoint: output.FLBPluginConfigKey(plugin, "minio_endpoint"), - access_key_id: output.FLBPluginConfigKey(plugin, "minio_access_key_id"), - secret_access_key: output.FLBPluginConfigKey(plugin, "minio_secret_access_key"), - use_ssl: use_ssl_bool, - bucket: output.FLBPluginConfigKey(plugin, "minio_bucket"), - } - fmt.Printf("[flb-files-metric] Initialized Minio\n") - return minio_config -} diff --git a/fluent_bit_plugins/src/go/out_files_metrics/storage_s3.go b/fluent_bit_plugins/src/go/out_files_metrics/storage_s3.go deleted file mode 100644 index 10da94378..000000000 --- a/fluent_bit_plugins/src/go/out_files_metrics/storage_s3.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "C" - "fmt" - - // "strconv" - "strings" - "unsafe" - - "github.com/fluent/fluent-bit-go/output" -) - -type S3Config struct { - endpoint string - access_key_id string - secret_access_key string - use_ssl bool - bucket string - src_fields []string - upload_fields []string -} - -// S3 -var s3_config S3Config -var s3_endpoint string -var s3_access_key_id string -var s3_secret_access_key string -var s3_use_ssl string -var s3_create_bucket string -var s3_bucket string -var s3_upload_fields string -var s3_src_fields string -var s3_split_upload_fields = make([]string, 0) -var s3_split_src_fields = make([]string, 0) - -func S3Init() error { - // Initialize s3 client object. - //TODO - - return nil -} - -func S3InitConf(plugin unsafe.Pointer) S3Config { - plugin_conf.storage.s3 = true - fmt.Printf("[flb-files-metric] Initializing S3...\n") - - s3_src_fields = output.FLBPluginConfigKey(plugin, "s3_src_fields") - s3_split_src_fields = strings.Fields(s3_src_fields) - s3_upload_fields = output.FLBPluginConfigKey(plugin, "s3_upload_fields") - s3_split_upload_fields = strings.Fields(s3_upload_fields) - - s3_config = S3Config{ - endpoint: output.FLBPluginConfigKey(plugin, "s3_endpoint"), - access_key_id: output.FLBPluginConfigKey(plugin, "s3_access_key_id"), - secret_access_key: output.FLBPluginConfigKey(plugin, "s3_secret_access_key"), - bucket: output.FLBPluginConfigKey(plugin, "s3_bucket"), - } - fmt.Printf("[flb-files-metric] Initialized S3\n") - return s3_config -} diff --git a/fluent_bit_plugins/src/go/out_minio/CMakeLists.txt b/fluent_bit_plugins/src/go/out_minio/CMakeLists.txt deleted file mode 100644 index e2bde57f2..000000000 --- a/fluent_bit_plugins/src/go/out_minio/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -cmake_minimum_required(VERSION 3.0) -project(out_minio) - -set(GO_COMPILER go) -set(MINIO_LIBRARY out_minio.so) -set(MINIO_INCLUDE out_minio.h) - -add_custom_target(plugin_out_minio_install ALL - COMMAND ${GO_COMPILER} get - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target(plugin_out_minio ALL - COMMAND ${GO_COMPILER} build -buildmode=c-shared -buildvcs=false -o ${MINIO_LIBRARY} ${CMAKE_CURRENT_SOURCE_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -install( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/${MINIO_LIBRARY} - FILES ${CMAKE_CURRENT_SOURCE_DIR}/${MINIO_INCLUDE} - DESTINATION lib -) diff --git a/fluent_bit_plugins/src/go/out_minio/go.mod b/fluent_bit_plugins/src/go/out_minio/go.mod deleted file mode 100644 index cacad1e0f..000000000 --- a/fluent_bit_plugins/src/go/out_minio/go.mod +++ /dev/null @@ -1,28 +0,0 @@ -module out_minio - -go 1.18 - -require ( - github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d - github.com/minio/minio-go/v7 v7.0.56 -) - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.2 // indirect - github.com/ugorji/go/codec v1.1.7 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect -) diff --git a/fluent_bit_plugins/src/go/out_minio/go.sum b/fluent_bit_plugins/src/go/out_minio/go.sum deleted file mode 100644 index ebb8900a7..000000000 --- a/fluent_bit_plugins/src/go/out_minio/go.sum +++ /dev/null @@ -1,53 +0,0 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d h1:b4a4JIzP5VT7l64NHnXF1nP95zRRN7wVMsD5D1jJiY0= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d/go.mod h1:L92h+dgwElEyUuShEwjbiHjseW410WIcNz+Bjutc8YQ= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY= -github.com/minio/minio-go/v7 v7.0.56/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/fluent_bit_plugins/src/go/out_minio/minio.go b/fluent_bit_plugins/src/go/out_minio/minio.go deleted file mode 100644 index 130d764fd..000000000 --- a/fluent_bit_plugins/src/go/out_minio/minio.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "strconv" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -var minio_client *minio.Client -var ctx = context.Background() -var endpoint string -var access_key_id string -var secret_access_key string -var use_ssl string -var create_bucket string -var bucket string - -func MinioInit() error { - - use_ssl_bool, err := strconv.ParseBool(use_ssl) - if err != nil { - fmt.Printf("[flb-minio] Could not parse ssl into boolean\n") - return err - } - // Initialize minio client object. - minio_client, err = minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(access_key_id, secret_access_key, ""), - Secure: use_ssl_bool, - }) - - if err != nil { - fmt.Printf("[flb-minio] Could not create client while creating bucket\n") - return err - } - - // Create the bucket - create_bucket, err := strconv.ParseBool(create_bucket) - if create_bucket { - err = minio_client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: "us-east-1"}) - if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := minio_client.BucketExists(ctx, bucket) - if errBucketExists == nil && exists { - // log.Printf("We already own %s\n", bucket) - } else { - fmt.Printf("[flb-minio] Could not create bucket\n") - return err - } - } else { - log.Printf("Successfully created %s\n", bucket) - } - } - return nil -} diff --git a/fluent_bit_plugins/src/go/out_minio/misc.go b/fluent_bit_plugins/src/go/out_minio/misc.go deleted file mode 100644 index 601916343..000000000 --- a/fluent_bit_plugins/src/go/out_minio/misc.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - "os" -) - -func NestedMapLookup(m map[interface{}]interface{}, ks ...string) (rval interface{}, err error) { - var ok bool - - if len(ks) == 0 { // degenerate input - return nil, fmt.Errorf("NestedMapLookup needs at least one key") - } - if rval, ok = m[ks[0]]; !ok { - return nil, fmt.Errorf("key not found; remaining keys: %v", ks) - } else if len(ks) == 1 { // we've reached the final key - return rval, nil - } else if m, ok = rval.(map[interface{}]interface{}); !ok { - return nil, fmt.Errorf("malformed structure at %#v", rval) - } else { // 1+ more keys - return NestedMapLookup(m, ks[1:]...) - } -} - -func get_file_content(path string) string { - file, err := os.Open(path) - - defer file.Close() - - if err != nil { - panic(err) - } - // Get the file content - buf := make([]byte, 512) - _, err = file.Read(buf) - - if err != nil { - panic(err) - } - - contentType := http.DetectContentType(buf) - - return contentType -} - -func contains(elems []int, v int) bool { - for _, s := range elems { - if v == s { - return true - } - } - return false -} diff --git a/fluent_bit_plugins/src/go/out_minio/out_minio.go b/fluent_bit_plugins/src/go/out_minio/out_minio.go deleted file mode 100644 index 3374b61f6..000000000 --- a/fluent_bit_plugins/src/go/out_minio/out_minio.go +++ /dev/null @@ -1,263 +0,0 @@ -package main - -import ( - "C" - "errors" - "fmt" - "os" - "strings" - "time" - "unsafe" - - "github.com/fluent/fluent-bit-go/output" - "github.com/minio/minio-go/v7" -) -import "strconv" - -var verbose string -var verbose_bool bool -var upload_fields string -var src_fields string -var groups string -var split_upload_fields = make([]string, 0) -var split_src_fields = make([]string, 0) -var split_groups = make([]string, 0) - -func getPath(record map[interface{}]interface{}, dot_separated_path string) string { - var key_lookup string = "" - var path string = "" - - if record["nested"] == true && record["flattened"] == false { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == true && record["flattened"] == true { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) + "/" + strings.Replace(dot_separated_path, ".", "/", -1) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == true && record["flattened"] == false { - key_lookup = "/" + fmt.Sprintf("%s", record["name"]) + "/" + strings.Replace(dot_separated_path, ".", "/", -1) - path = fmt.Sprintf("%s", record[key_lookup]) - } else if record["nested"] == false && record["flattened"] == false { - local_path_val, err := NestedMapLookup(record, strings.Split(dot_separated_path, ".")...) - if err == nil { - path = fmt.Sprintf("%s", local_path_val) - } - } - - return path -} - -//export FLBPluginFlush -func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { - init_err := MinioInit() - - if init_err != nil { - // fmt.Printf("[flb-minio] Could not connect to minio\n") - return output.FLB_RETRY - } - fmt.Printf("[flb-minio] Connected to Minio\n") - var ret int - var ts interface{} - var record map[interface{}]interface{} - - // Create Fluent Bit decoder - dec := output.NewDecoder(data, int(length)) - retry := false - - for { - // Extract Record - ret, ts, record = output.GetRecord(dec) - if ret != 0 || ts == nil { - fmt.Printf("[flb-minio] Could not get records, ts = %d, ret = %d\n", ts, ret) - break - } - if ret != 0 { - // fmt.Printf("[flb-minio] No more records, ret = %d\n", ret) - break - } - if verbose_bool || true { - fmt.Printf("[flb-minio] Got records!\n") - // Print record keys and values - fmt.Printf("%s: {", C.GoString(tag)) - for k, v := range record { - fmt.Printf("\"%s\": %v, ", k, v) - } - fmt.Printf("}\n") - } - - var current_group string - - // Indexes of paths to ignore - index_ignore := []int{} - src_paths := []string{} - index := 0 - - for split_src_field_i, split_src_field_v := range split_src_fields { - fmt.Printf("[flb-minio] current_group=%s\n", current_group) - // Set group if it is not yet done - - var local_path string = "" - if len(current_group) == 0 { - local_path = getPath(record, split_src_field_v) - fmt.Printf("[flb-minio] Found %s\n", local_path) - - // Add to group if the file exists - if len(local_path) != 0 { - if _, err := os.Stat(local_path); errors.Is(err, os.ErrNotExist) { - // File does not exist - fmt.Printf("[flb-minio] File %s does not exist\n", local_path) - } else { - current_group = fmt.Sprintf("%s", record["name"]) - fmt.Printf("[flb-minio] File %s exists, setting group to %s\n", local_path, current_group) - } - } - } - if current_group == fmt.Sprintf("%s", record["name"]) { - fmt.Printf("[flb-minio] split_src_field_i match index %d: current_group=%s, split_src_field_v=%s\n", split_src_field_i, current_group, split_src_field_v) - local_path = getPath(record, split_src_field_v) - - // Add to group if the file exists - if len(local_path) != 0 { - if _, err := os.Stat(local_path); errors.Is(err, os.ErrNotExist) { - // File does not exist - index_ignore = append(index_ignore, index) - } else { - src_paths = append(src_paths, fmt.Sprintf("%s", local_path)) - } - } - } - index++ - } - - index = 0 - upload_paths := []string{} - for split_upload_field_i, split_upload_field_v := range split_upload_fields { - // Ignore fields that don't belong to the group. Since they are not in the JSON, we skip them to not create errors - if current_group == fmt.Sprintf("%s", record["name"]) { - fmt.Printf("[flb-minio] split_upload_field_i match index %d: current_group=%s\n", split_upload_field_i, current_group) - // Skip indexes of files that don't exist - if !contains(index_ignore, index) { - var upload_path = getPath(record, split_upload_field_v) - fmt.Printf("[flb-minio] Adding path %s in list\n", upload_path) - if len(upload_path) != 0 { - upload_paths = append(upload_paths, upload_path) - } - } - } - index++ - } - - size := len(src_paths) - fmt.Println("[flb-minio] Files to upload:", size) - for _, path := range upload_paths { - fmt.Println(path) - } - - for src_paths_i, src_paths_v := range src_paths { - // Upload the file file with FPutObject - fmt.Printf("[flb-minio] Uploading %s to %s, format: %s\n", src_paths_v, upload_paths[src_paths_i], get_file_content(src_paths_v)) - - // Get the initial modification time - file, err := os.Open(src_paths_v) - if err != nil { - fmt.Println(err) - retry = true - break - } - defer file.Close() - initialModTime, err := file.Stat() - if err != nil { - fmt.Println(err) - retry = true - break - } - fmt.Printf("[flb-minio] Initial mode: %s", initialModTime) - // Keep looping until the file is no longer being written to - for { - time.Sleep(20 * time.Millisecond) - - // Get the current modification time - currentModTime, err := file.Stat() - if err != nil { - fmt.Println(err) - retry = true - break - } - - // Compare the modification times - if !currentModTime.ModTime().Equal(initialModTime.ModTime()) { - fmt.Println("[flb-minio] The file is being written to.") - initialModTime = currentModTime - } else { - fmt.Println("[flb-minio] The file is no longer being written to.") - _, err = minio_client.FPutObject(ctx, bucket, upload_paths[src_paths_i], src_paths_v, minio.PutObjectOptions{ContentType: get_file_content(src_paths_v)}) - if err != nil { - fmt.Printf("[flb-minio] Could not upload %s to %s, format: %s\n", src_paths_v, upload_paths[src_paths_i], get_file_content(src_paths_v)) - fmt.Println(err) - retry = true - } else { - fmt.Printf("[flb-minio] Uploaded %s to %s, format: %s\n", src_paths_v, upload_paths[src_paths_i], get_file_content(src_paths_v)) - break - } - } - } - } - } - - if retry == true { - return output.FLB_RETRY - } - return output.FLB_OK -} - -//export FLBPluginRegister -func FLBPluginRegister(def unsafe.Pointer) int { - return output.FLBPluginRegister(def, "minio", "Minio GO!") -} - -// (fluentbit will call this) -// plugin (context) pointer to fluentbit context (state/ c code) -// -//export FLBPluginInit -func FLBPluginInit(plugin unsafe.Pointer) int { - verbose = output.FLBPluginConfigKey(plugin, "verbose") - var err error - verbose_bool, err = strconv.ParseBool(verbose) - if err != nil { - fmt.Printf("[flb-minio] Could not parse verbose into boolean\n") - return output.FLB_RETRY - } - fmt.Printf("[flb-minio] verbose = '%t'\n", verbose_bool) - - endpoint = output.FLBPluginConfigKey(plugin, "endpoint") - fmt.Printf("[flb-minio] endpoint = '%s'\n", endpoint) - access_key_id = output.FLBPluginConfigKey(plugin, "access_key_id") - fmt.Printf("[flb-minio] access_key_id = '%s'\n", access_key_id) - secret_access_key = output.FLBPluginConfigKey(plugin, "secret_access_key") - fmt.Printf("[flb-minio] secret_access_key = '%s'\n", secret_access_key) - use_ssl = output.FLBPluginConfigKey(plugin, "use_ssl") - fmt.Printf("[flb-minio] use_ssl = '%s'\n", use_ssl) - create_bucket = output.FLBPluginConfigKey(plugin, "create_bucket") - fmt.Printf("[flb-minio] create_bucket = '%s'\n", create_bucket) - bucket = output.FLBPluginConfigKey(plugin, "bucket") - fmt.Printf("[flb-minio] bucket = '%s'\n", bucket) - upload_fields = output.FLBPluginConfigKey(plugin, "upload_fields") - fmt.Printf("[flb-minio] upload_fields = '%s'\n", upload_fields) - src_fields = output.FLBPluginConfigKey(plugin, "src_fields") - fmt.Printf("[flb-minio] src_fields = '%s'\n", src_fields) - - split_upload_fields = strings.Fields(upload_fields) - fmt.Printf("[flb-minio] split_upload_fields = '%v'\n", split_upload_fields) - split_src_fields = strings.Fields(src_fields) - fmt.Printf("[flb-minio] split_src_fields = '%v'\n", split_src_fields) - - return output.FLB_OK -} - -//export FLBPluginExit -func FLBPluginExit() int { - fmt.Printf("[flb-minio] Exited plugin\n") - return output.FLB_OK -} - -func main() { -} diff --git a/fluent_bit_vendor/CMakeLists.txt b/fluent_bit_vendor/CMakeLists.txt deleted file mode 100644 index 4df1b2d80..000000000 --- a/fluent_bit_vendor/CMakeLists.txt +++ /dev/null @@ -1,209 +0,0 @@ -cmake_minimum_required(VERSION 3.5) - -project(fluent_bit_vendor) - -find_package(ament_cmake REQUIRED) - -set(cmake_commands) -set(cmake_configure_args -Wno-dev) - -if(WIN32) - if(DEFINED CMAKE_GENERATOR) - list(APPEND cmake_configure_args -G ${CMAKE_GENERATOR}) - endif() - if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "^(x86_|x86-|AMD|amd|x)64$") - list(APPEND cmake_configure_args -A x64) - endif() -endif() - -if(DEFINED CMAKE_BUILD_TYPE) - if(WIN32) - build_command(_build_command CONFIGURATION ${CMAKE_BUILD_TYPE}) - list(APPEND cmake_commands "BUILD_COMMAND ${_build_command}") - else() - list(APPEND cmake_configure_args -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) - endif() -endif() - -if(DEFINED CMAKE_TOOLCHAIN_FILE) - list(APPEND cmake_configure_args "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") - if(ANDROID) - if(DEFINED ANDROID_ABI) - list(APPEND cmake_configure_args "-DANDROID_ABI=${ANDROID_ABI}") - endif() - if(DEFINED ANDROID_CPP_FEATURES) - list(APPEND cmake_configure_args "-DANDROID_CPP_FEATURES=${ANDROID_CPP_FEATURES}") - endif() - if(DEFINED ANDROID_FUNCTION_LEVEL_LINKING) - list(APPEND cmake_configure_args "-DANDROID_FUNCTION_LEVEL_LINKING=${ANDROID_FUNCTION_LEVEL_LINKING}") - endif() - if(DEFINED ANDROID_NATIVE_API_LEVEL) - list(APPEND cmake_configure_args "-DANDROID_NATIVE_API_LEVEL=${ANDROID_NATIVE_API_LEVEL}") - endif() - if(DEFINED ANDROID_NDK) - list(APPEND cmake_configure_args "-DANDROID_NDK=${ANDROID_NDK}") - endif() - if(DEFINED ANDROID_STL) - list(APPEND cmake_configure_args "-DANDROID_STL=${ANDROID_STL}") - endif() - if(DEFINED ANDROID_TOOLCHAIN_NAME) - list(APPEND cmake_configure_args "-DANDROID_TOOLCHAIN_NAME=${ANDROID_TOOLCHAIN_NAME}") - endif() - endif() -else() - list(APPEND cmake_configure_args "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") -endif() -set(fluent_bit_external_project_dir ${CMAKE_CURRENT_BINARY_DIR}/fluent_bit) -include(ExternalProject) -set(fluent_bit_version "fix-2.1.3") - -ExternalProject_Add(fluent_bit_src - GIT_REPOSITORY https://github.com/minipada/fluent-bit - GIT_TAG ${fluent_bit_version} - GIT_CONFIG advice.detachedHead=false - PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_LIST_DIR}/mpack.patch - # Suppress git update due to https://gitlab.kitware.com/cmake/cmake/-/issues/16419 - UPDATE_COMMAND "" - TIMEOUT 6000 - BUILD_IN_SOURCE On - ${cmake_commands} - CMAKE_ARGS - -DBUILD_TESTING=Off - -DCMAKE_INSTALL_PREFIX=${fluent_bit_external_project_dir}/install/ - -DCMAKE_INSTALL_LIBDIR=lib/ - -DFLB_EXAMPLES=Off - -DFLB_OUT_PGSQL=On - -DFLB_SHARED_LIB=On - -DFLB_PROXY_GO=On - -DSYSTEMD_UNITDIR=${fluent_bit_external_project_dir}/install/system - ${cmake_configure_args} -) - -externalproject_get_property(fluent_bit_src INSTALL_DIR) -set(fluent_bit_INCLUDE_DIR "${INSTALL_DIR}/../fluent_bit/install/include/") -set(fluent_bit_LIB_DIR "${INSTALL_DIR}/../fluent_bit/install/lib/fluent-bit") - -set(FB_SRC_LIB_INCLUDE_DIR "${INSTALL_DIR}/src/fluent_bit_src/lib") -set(FB_SRC_INCLUDE_DIR "${INSTALL_DIR}/src/fluent_bit_src/include/fluent-bit") - -set(ares_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/c-ares-1.19.0/include/") -set(cfl_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/cfl/include/cfl/") -set(config_format_INCLUDE_DIR "${FB_SRC_INCLUDE_DIR}/config_format/") -set(cmetrics_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/cmetrics/include/cmetrics/") -set(ctraces_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/ctraces/include/ctraces/") -set(jsmn_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/jsmn/") -set(mkcore_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/monkey/include/monkey/mk_core/") -set(mpack_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/mpack-amalgamation-1.1/src/mpack/") -set(msgpack_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/msgpack-c/include/msgpack/") -set(msgpack_INCLUDE_FILES "${FB_SRC_LIB_INCLUDE_DIR}/msgpack-c/include/msgpack.h") -set(prometheus_remote_write_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/cmetrics/include/prometheus_remote_write/") -set(xxhash_INCLUDE_DIR "${FB_SRC_LIB_INCLUDE_DIR}/cfl/lib/xxhash/") - -install( - DIRECTORY - ${fluent_bit_INCLUDE_DIR} - ${ares_INCLUDE_DIR} - DESTINATION - include -) - -install( - DIRECTORY - ${cfl_INCLUDE_DIR} - DESTINATION - include/cfl -) -install( - DIRECTORY - ${config_format_INCLUDE_DIR} - DESTINATION - include/config_format -) -install( - DIRECTORY - ${cmetrics_INCLUDE_DIR} - DESTINATION - include/cmetrics -) -install( - DIRECTORY - ${ctraces_INCLUDE_DIR} - DESTINATION - include/ctraces -) -install( - DIRECTORY - ${jsmn_INCLUDE_DIR} - DESTINATION - include/jsmn -) -install( - DIRECTORY - ${mkcore_INCLUDE_DIR} - DESTINATION - include/mk_core -) -install( - DIRECTORY - ${mpack_INCLUDE_DIR} - DESTINATION - include/mpack -) -install( - DIRECTORY - ${msgpack_INCLUDE_DIR} - DESTINATION - include/msgpack -) -install( - FILES - ${msgpack_INCLUDE_FILES} - DESTINATION - include -) -install( - DIRECTORY - ${prometheus_remote_write_INCLUDE_DIR} - DESTINATION - include/prometheus_remote_write -) -install( - DIRECTORY - ${xxhash_INCLUDE_DIR} - DESTINATION - include -) -install( - FILES - ${fluent_bit_LIB_DIR}/libfluent-bit.so - DESTINATION - lib -) -install( - FILES - ${INSTALL_DIR}/src/fluent_bit_src/library/libmsgpack-c-static.a - DESTINATION - lib -) -install( - FILES - ${INSTALL_DIR}/src/fluent_bit_src/library/libfluent-bit.a - DESTINATION - lib -) -install( - DIRECTORY - cmake - DESTINATION - share/${PROJECT_NAME} -) -install( - DIRECTORY - ${INSTALL_DIR}/src/fluent_bit_src/cmake - DESTINATION - share/${PROJECT_NAME} -) - -ament_export_include_directories(include) -ament_export_libraries(libfluent-bit.so fluent-bit) -ament_package(CONFIG_EXTRAS fluentbit_vendor-extras.cmake) diff --git a/fluent_bit_vendor/COLCON_IGNORE b/fluent_bit_vendor/COLCON_IGNORE deleted file mode 100644 index 045fb9b16..000000000 --- a/fluent_bit_vendor/COLCON_IGNORE +++ /dev/null @@ -1,3 +0,0 @@ -Ignored on the jazzy line pending the DC 2.0 embedded-Fluent-Bit demolition (ADR-0001: -replace the embedded Fluent Bit engine with an external Vector shipper). See the dc-2.0 -epic (#241). Do not delete this package; it is dropped, not removed. diff --git a/fluent_bit_vendor/cmake/Modules/Findfluent_bit.cmake b/fluent_bit_vendor/cmake/Modules/Findfluent_bit.cmake deleted file mode 100644 index 40214725e..000000000 --- a/fluent_bit_vendor/cmake/Modules/Findfluent_bit.cmake +++ /dev/null @@ -1,38 +0,0 @@ -if(NOT fluent_bit_ROOT_DIR AND DEFINED ENV{fluent_bit_ROOT_DIR}) - set(fluent_bit_ROOT_DIR "$ENV{fluent_bit_ROOT_DIR}" CACHE PATH - "fluent_bit base directory location (optional, used for nonstandard installation paths)") -endif() - -set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a") -if(fluent_bit_ROOT_DIR) - set(fluent_bit_INCLUDE_PATH PATHS "${fluent_bit_ROOT_DIR}/include" NO_DEFAULT_PATH) - set(fluent_bit_LIBRARY_PATH PATHS "${fluent_bit_ROOT_DIR}/lib/fluent-bit" NO_DEFAULT_PATH) -else() - set(fluent_bit_INCLUDE_PATH "") - set(fluent_bit_LIBRARY_PATH "") -endif() - -# Search for headers and the library -find_path(fluent_bit_INCLUDE_DIR NAMES "fluent-bit.h" ${fluent_bit_INCLUDE_PATH}) -find_library(fluent_bit_LIBRARY NAMES fluent-bit ${fluent_bit_LIBRARY_PATH}) - -mark_as_advanced(fluent_bit_INCLUDE_DIR fluent_bit_LIBRARY) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - fluent_bit - DEFAULT_MSG - fluent_bit_INCLUDE_DIR - fluent_bit_LIBRARY -) - -if(${fluent_bit_FOUND}) - set(fluent_bit_INCLUDE_DIRS ${fluent_bit_INCLUDE_DIR} ${msgpack_INCLUDE_DIR} ${monkey_INCLUDE_DIR} ${cmetrics_INCLUDE_DIR} ${prometheus_remote_write_INCLUDE_DIR}) - set(fluent_bit_LIBRARIES ${fluent_bit_LIBRARY}) - - add_library(fluent_bit::fluent_bit UNKNOWN IMPORTED) - set_property(TARGET fluent_bit::fluent_bit PROPERTY IMPORTED_LOCATION ${fluent_bit_LIBRARY} ${msgpack_INCLUDE_DIR}) - set_property(TARGET fluent_bit::fluent_bit PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${fluent_bit_INCLUDE_DIR} ${msgpack_INCLUDE_DIR} ${monkey_INCLUDE_DIR} ${cmetrics_INCLUDE_DIR} ${prometheus_remote_write_INCLUDE_DIR}) - list(APPEND fluent_bit_TARGETS fluent_bit::fluent_bit) - -endif() diff --git a/fluent_bit_vendor/fluentbit_vendor-extras.cmake b/fluent_bit_vendor/fluentbit_vendor-extras.cmake deleted file mode 100644 index a4f2669b0..000000000 --- a/fluent_bit_vendor/fluentbit_vendor-extras.cmake +++ /dev/null @@ -1,3 +0,0 @@ -list(INSERT CMAKE_MODULE_PATH 0 "${fluent_bit_vendor_DIR}/Modules") -list(INSERT CMAKE_MODULE_PATH 0 "${fluent_bit_vendor_DIR}/cmake") -list(INSERT CMAKE_MODULE_PATH 0 "${fluent_bit_vendor_DIR}/sanitizers-cmake/cmake") diff --git a/fluent_bit_vendor/mpack.patch b/fluent_bit_vendor/mpack.patch deleted file mode 100644 index 9394177cd..000000000 --- a/fluent_bit_vendor/mpack.patch +++ /dev/null @@ -1,356 +0,0 @@ -diff --git a/include/fluent-bit/flb_lib.h b/include/fluent-bit/flb_lib.h -index 7e9275297..1ce70cfab 100644 ---- a/include/fluent-bit/flb_lib.h -+++ b/include/fluent-bit/flb_lib.h -@@ -22,6 +22,7 @@ - - #include - #include -+#include - - /* Lib engine status */ - #define FLB_LIB_ERROR -1 -@@ -79,4 +80,10 @@ FLB_EXPORT int flb_loop(flb_ctx_t *ctx); - FLB_EXPORT int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len); - FLB_EXPORT int flb_lib_config_file(flb_ctx_t *ctx, const char *path); - -+/* Flb plugin */ -+FLB_EXPORT int flb_plugin_load_wr(char *path, struct flb_plugins *ctx, struct flb_config *config); -+FLB_EXPORT int flb_plugin_load_router_wr(char *path, struct flb_config *config); -+FLB_EXPORT int flb_plugin_load_config_file_wr(const char *file, struct flb_config *config); -+FLB_EXPORT void flb_plugin_destroy_wr(struct flb_plugins *ctx); -+ - #endif -diff --git a/lib/cmetrics/lib/mpack/src/mpack/mpack.h b/lib/cmetrics/lib/mpack/src/mpack/mpack.h -index 7c0b3f17e..cfe860eb4 100644 ---- a/lib/cmetrics/lib/mpack/src/mpack/mpack.h -+++ b/lib/cmetrics/lib/mpack/src/mpack/mpack.h -@@ -3271,129 +3271,6 @@ MPACK_EXTERN_C_END - #undef mpack_write_kv - #endif - --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int8_t value) { -- mpack_write_i8(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int16_t value) { -- mpack_write_i16(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int32_t value) { -- mpack_write_i32(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int64_t value) { -- mpack_write_i64(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint8_t value) { -- mpack_write_u8(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint16_t value) { -- mpack_write_u16(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint32_t value) { -- mpack_write_u32(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint64_t value) { -- mpack_write_u64(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, bool value) { -- mpack_write_bool(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, float value) { -- mpack_write_float(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, double value) { -- mpack_write_double(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, char *value) { -- mpack_write_cstr_or_nil(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, const char *value) { -- mpack_write_cstr_or_nil(writer, value); --} -- --/* C++ generic write for key-value pairs */ -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int8_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i8(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int16_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i16(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int32_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i32(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int64_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i64(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint8_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u8(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint16_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u16(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint32_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u32(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint64_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u64(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, bool value) { -- mpack_write_cstr(writer, key); -- mpack_write_bool(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, float value) { -- mpack_write_cstr(writer, key); -- mpack_write_float(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, double value) { -- mpack_write_cstr(writer, key); -- mpack_write_double(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, char *value) { -- mpack_write_cstr(writer, key); -- mpack_write_cstr_or_nil(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, const char *value) { -- mpack_write_cstr(writer, key); -- mpack_write_cstr_or_nil(writer, value); --} -- --/** -- * @} -- */ -- - #endif /* __cplusplus */ - - /** -diff --git a/lib/mpack-amalgamation-1.1/src/mpack/mpack.h b/lib/mpack-amalgamation-1.1/src/mpack/mpack.h -index 803f03e41..7c2ceb338 100644 ---- a/lib/mpack-amalgamation-1.1/src/mpack/mpack.h -+++ b/lib/mpack-amalgamation-1.1/src/mpack/mpack.h -@@ -4104,7 +4104,7 @@ MPACK_INLINE void mpack_finish_type(mpack_writer_t* writer, mpack_type_t type) { - * @} - */ - --#if MPACK_HAS_GENERIC && !defined(__cplusplus) -+#if MPACK_HAS_GENERIC - - /** - * @name Type-Generic Writers -@@ -4183,146 +4183,6 @@ MPACK_INLINE void mpack_finish_type(mpack_writer_t* writer, mpack_type_t type) { - // The rest of this file contains C++ overloads, so we end extern "C" here. - MPACK_EXTERN_C_END - --#if defined(__cplusplus) || defined(MPACK_DOXYGEN) -- --/** -- * @name C++ write overloads -- * @{ -- */ -- --/* -- * C++ generic writers for primitive values -- */ -- --#ifdef MPACK_DOXYGEN --#undef mpack_write --#undef mpack_write_kv --#endif -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int8_t value) { -- mpack_write_i8(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int16_t value) { -- mpack_write_i16(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int32_t value) { -- mpack_write_i32(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, int64_t value) { -- mpack_write_i64(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint8_t value) { -- mpack_write_u8(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint16_t value) { -- mpack_write_u16(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint32_t value) { -- mpack_write_u32(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint64_t value) { -- mpack_write_u64(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, bool value) { -- mpack_write_bool(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, float value) { -- mpack_write_float(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, double value) { -- mpack_write_double(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, char *value) { -- mpack_write_cstr_or_nil(writer, value); --} -- --MPACK_INLINE void mpack_write(mpack_writer_t* writer, const char *value) { -- mpack_write_cstr_or_nil(writer, value); --} -- --/* C++ generic write for key-value pairs */ -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int8_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i8(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int16_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i16(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int32_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i32(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int64_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_i64(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint8_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u8(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint16_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u16(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint32_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u32(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint64_t value) { -- mpack_write_cstr(writer, key); -- mpack_write_u64(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, bool value) { -- mpack_write_cstr(writer, key); -- mpack_write_bool(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, float value) { -- mpack_write_cstr(writer, key); -- mpack_write_float(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, double value) { -- mpack_write_cstr(writer, key); -- mpack_write_double(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, char *value) { -- mpack_write_cstr(writer, key); -- mpack_write_cstr_or_nil(writer, value); --} -- --MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, const char *value) { -- mpack_write_cstr(writer, key); -- mpack_write_cstr_or_nil(writer, value); --} -- --/** -- * @} -- */ -- --#endif /* __cplusplus */ - - /** - * @} -diff --git a/src/flb_lib.c b/src/flb_lib.c -index 996816e23..a81c64924 100644 ---- a/src/flb_lib.c -+++ b/src/flb_lib.c -@@ -24,6 +24,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -66,6 +67,26 @@ static inline int flb_socket_init_win32(void) - } - #endif - -+int flb_plugin_load_wr(char *path, struct flb_plugins *ctx, struct flb_config *config) -+{ -+ return flb_plugin_load(path,ctx,config); -+} -+ -+int flb_plugin_load_router_wr(char *path, struct flb_config *config) -+{ -+ return flb_plugin_load_router(path, config); -+} -+ -+int flb_plugin_load_config_file_wr(const char *file, struct flb_config *config) -+{ -+ return flb_plugin_load_config_file(file, config); -+} -+ -+void flb_plugin_destroy_wr(struct flb_plugins *ctx) -+{ -+ flb_plugin_destroy(ctx); -+} -+ - static inline struct flb_input_instance *in_instance_get(flb_ctx_t *ctx, - int ffd) - { diff --git a/fluent_bit_vendor/package.xml b/fluent_bit_vendor/package.xml deleted file mode 100644 index b06ba8782..000000000 --- a/fluent_bit_vendor/package.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - fluent_bit_vendor - 2.0.0 - A vendor package for An End to End Observability Pipeline - David Bensoussan - MPL-2.0 - - ament_cmake - dc_interfaces - git - flex - bison - libpq-dev - libssl-dev - rclc - - - ament_cmake - - diff --git a/progress.txt b/progress.txt index 6824ca7cd..5c44cd928 100644 --- a/progress.txt +++ b/progress.txt @@ -914,3 +914,136 @@ package") and rewrote dc_bridge in C++. ADR-0007 records the decision and ration prebuilt aws-sdk-cpp to avoid the ~20-min source build; GCS/Azure ObjectStore implementations (S3 only today, matching the Rust); the #268 gz-sim port (dc_demos/dc_simulation still COLCON_IGNORE'd, unrelated to the language change). + +### #250 - DC 2.0 S9: Demolition — removed Fluent Bit packages and the flb_* layer + +Deleted every package the embedded-Fluent-Bit architecture left behind, now that the +Bridge + Vector data plane (#242-#249) fully replaces it, per ADRs 0001/0003 and #241's +"Deletions are part of the deliverable" note: + +- **Deleted outright**: `fluent_bit_vendor` (the forked Fluent Bit source build), + `fluent_bit_plugins` (the C `in_ros2` input plugin and the two cgo-built Go output + plugins — the last Go code in the repo), and the entire `dc_destinations` package + (the `destination_server` node, all twelve `flb_*` pluginlib Destination plugins, and + `rcl.cpp` — `dc_destinations` was 100% Fluent-Bit-shaped: every plugin took a + `flb_ctx_t*` and linked `fluent_bit::fluent_bit`, so there was nothing in it worth + keeping standalone). Also deleted `dc_core/include/dc_core/destination.hpp` (the + `dc_core::Destination` pluginlib base class every `flb_*` plugin implemented — dead + the moment `dc_destinations` is gone; confirmed via `codegraph`/grep no other package + included it). +- **Go toolchain removed**: the `tekwizely/pre-commit-golang` `go-fmt` hook (nothing + left to format) and the now-dead `fluent_bit_plugins/src/go/*/*.so`/`.h` + `.gitignore` entries. +- **Build/CI plumbing**: `.dockerignore`'s `dc_destinations`/`fluent_bit_plugins`/ + `fluent_bit_vendor` `package.xml` allowlist entries removed (nothing left to + allowlist); `tools/e2e/Containerfile`'s comment explaining `dc_demos`/`dc_simulation`'s + `COLCON_IGNORE` no longer cites `dc_destinations/fluent_bit_*` as a parallel example + (they're deleted now, not ignored); `dc_bringup/launch/dc_bringup.launch.py`'s two + comments explaining why no `destination_server` node is launched were reworded from + "COLCON_IGNOREd pending the demolition slice" (forward-looking, #242-era) to stating + plainly that `dc_destinations` was removed in #250 — the launch file itself needed no + behavior change since `dc_bridge` already fully replaced `destination_server` starting + in #244/#247. +- **Requirements**: `requirements/destination_plugins.sdoc` (twelve `DP-PLU-*` + Fluent-Bit-output requirements) and `requirements/destination_server.sdoc` (the + pluginlib-loading `DS-*` requirements, including a stray Redis-backend requirement + that was never implemented either) deleted — both specified a subsystem that no + longer exists; confirmed via grep no other `.sdoc` file cross-references their UIDs + (`strictdoc export requirements` needs no per-file registration, so removal is safe). +- **Docs — updated, not just stripped of dead links**: `doc/src/SUMMARY.md` and + `doc/src/dc/destinations.md` lost the thirteen `flb_*`/`rcl` destination sub-pages + (deleted, since `dc_destinations` is gone and DC 2.0's blessed-Destinations contract + is already fully documented above the old "Legacy (Humble)" section, which is also + removed — the humble branch's own tree documents that architecture natively, so + duplicating it here now that no code on this branch runs it serves no one). + `data_pipeline.md` lost its "Legacy (Humble): embedded Fluent Bit pipeline" section + (mermaid diagram included) for the same reason. `concepts.md`, `configuration_examples.md`, + `faq.md`, `groups.md`, `introduction.md`, and `measurements.md` were **still written + entirely around the Fluent-Bit/destination_server architecture in the present tense** + (not yet touched by any earlier DC 2.0 doc pass) — rewrote the Fluent-Bit-specific + sections/examples in each for the Bridge/Vector model (`dc_bridge`'s `destinations` + list replaces `destination_server`'s `destination_plugins`, Destination names replace + `flb_*` plugin names, Vector's disk buffer replaces Fluent Bit's memory+filesystem + buffering, the passthrough via `custom_config_files` replaces "write a C/Go Fluent Bit + plugin" as the extension story). All fourteen `doc/src/dc/measurements/*.md` pages' + identical `tags: ["flb_stdout"]` example line became `tags: ["console"]` (bulk `sed`, + verified identical pattern via grep first); `camera.md`'s `flb_minio` destination + example block was rewritten as an `s3`/`receives: files` Destination block matching + the real `dc_bridge` uploader contract in `destinations.md`. +- **`dc_measurements/include/dc_measurements/measurement.hpp`**: three identical + comments referencing "the destination_server" (describing why untagged data is still + published but not delivered) reworded to "the Bridge" — the only C++ comment fallout + found outside the deleted packages themselves (grepped for `dc_destinations`/ + `destination_server`/`DestinationServer` across all `.cpp`/`.hpp`/`CMakeLists.txt`/ + `package.xml` after the deletions). +- **Deliberately out of scope, left tracked separately**: `dc_demos/params/*.yaml` and + `dc_demos/launch/*.launch.py` (still `flb_*`-shaped — group tags like + `["flb_stdout"]`, a `destination_server:`/`flb:` params block) and + `doc/src/dc/demos/*.md` (their documentation) — issue #251 ("DC 2.0 S10: Demos rework + on the new pipeline") exists specifically for this and is `Blocked by #250`, so + reworking those params here would duplicate that issue's own acceptance criteria. + Likewise `tools/infrastructure/docker/` (the standalone PostgreSQL/MinIO/Grafana + compose demo stack, whose Grafana dashboards reference an `flb_influxdb` datasource + UID) is #251's "docker-compose demo infrastructure" acceptance criterion, not + touched here. `CLAUDE.md`'s and `CONTEXT.md`'s own Fluent-Bit mentions are + intentional comparative/historical framing (e.g. CONTEXT.md's glossary note that "no + Fluentd or Fluent Bit software runs anywhere in DC 2.0"), and the ADRs + (`docs/adr/000{1,2,3,7}-*.md`) and `progress.txt`'s own prior entries are immutable + decision/history records — none of these were edited. +- **Verification**: a tree-wide `grep -rlI` for `fluent_bit|flb_|FLB_|FluentBit|Fluent + Bit` after all edits returns only the intentionally-deferred/historical files listed + above. Ran a real `colcon build` via `tools/e2e/scripts/build.sh` (Podman, the same + harness #249 built) against the full workspace, reusing that session's cached + `dc-workspace` image layers — rosdep resolved and installed the full dependency set + with no complaint about the removed packages (no dangling `dc_destinations`/ + `fluent_bit_*` depends anywhere, confirmed by grep beforehand too). **All 14 + remaining `dc_*`/vendor packages built clean** (`aws_sdk_vendor`, `dc_bridge`, + `dc_bringup`, `dc_cli`, `dc_common`, `dc_core`, `dc_description`, `dc_group`, + `dc_interfaces`, `dc_lifecycle_manager`, `dc_measurements`, `dc_services`, + `dc_util`, `vector_vendor` — `dc_demos`/`dc_simulation` stay `COLCON_IGNORE`d for + the unrelated Gazebo-Classic reason, #268). Then started the CI-equivalent + Postgres+RustFS test stores (`podman compose -f tools/e2e/compose.test.yaml up -d`, + `PODMAN_COMPOSE_PROVIDER=podman-compose` — the docker-compose cli-plugin fallback + needs a podman API socket that isn't running in this sandbox) and ran `tools/e2e/ + scripts/test.sh` (the same script `ci.yaml` calls): **`colcon test` — 84 tests, 0 + errors, 0 failures, 0 skipped**, including `dc_bridge`'s store-backed tests against + the real stores (which hard-fail rather than skip per the #246 follow-up decision, + so this run genuinely exercised them, not a skip path). Stores torn down + afterward. This closes acceptance criteria 3 ("Full colcon build and all tests + pass after removal") and 4 ("Go toolchain no longer required") for real, not just + by inspection. `pre-commit` was not re-run in this sandbox (same environment-gap + skips as every prior DC 2.0 slice: `build-doc`, `poetry-requirements`, `pycln`, + `flake8`). + +### #250 follow-up — fixed a real (pre-existing, unrelated) flaky test found by real CI + +The GitHub Actions run for this PR's rebased-for-DCO commit failed `colcon-test` once +on `dc_bridge/test/supervisor_test.cpp`'s `Supervisor.RestartsProcessThatExitsOnItsOwn` +— `is_running()` was still `true` after a fixed 300ms `sleep_for()` that assumed the +supervised child (`sh -c "sleep 0.05"`) would always be forked, exited, and reaped well +within that margin. `dc_bridge` has zero diff on this branch (this PR never touches +it), the same test passed in this session's own local `colcon test` run (84/84) and in +the prior successful `jazzy` CI run with identical code — pointing at CI-runner +scheduling jitter (right after a full workspace build, on a shared 2-core runner) as +the cause, not a real regression. + +- Root-caused rather than just retried: `RestartsProcessThatExitsOnItsOwn` and + `RespectsRestartBackoff` both asserted on the child's exit via a fixed + `sleep_for(200-300ms)` then a single check — inherently flaky under scheduler + jitter, and unnecessary, since the same test file already has the correct pattern + (`SupervisedProcessDiesWithItsSpawner`'s poll-with-deadline loop). Added a small + `wait_until(pred, timeout=5s)` helper (polls every 10ms) and switched both tests to + it — same final assertions, just no longer time-boxed by a wall-clock guess. +- Verified the fix for real, not just by re-reading it: incrementally rebuilt + `dc_bridge` inside the exact `dc-workspace:latest` Podman image CI uses (bind-mounted + updated source over the already-built image, `colcon build --packages-select + dc_bridge`), then ran the `supervisor_test` binary directly 30 times in a row — 30/30 + clean, and the fixed-margin tests now finish in ~60-80ms instead of always paying the + full 200-300ms fixed sleep. +- Re-ran the failed GitHub Actions `colcon-test` job once (confirming the original + failure was transient, not reproducible) before pushing this fix; a fresh full CI run + against the fixed commit was kicked off to confirm end-to-end. +- `clang-format` isn't installed in this sandbox and the Podman workspace image doesn't + carry it either (not part of the C++ toolchain it installs) — not run; the added code + was hand-matched to the file's existing style (2-space indent, brace-on-its-own-line, + same `namespace { … }` shape) instead. diff --git a/requirements/destination_plugins.sdoc b/requirements/destination_plugins.sdoc deleted file mode 100644 index 7e9aa2efc..000000000 --- a/requirements/destination_plugins.sdoc +++ /dev/null @@ -1,332 +0,0 @@ -[DOCUMENT] -TITLE: Destination plugins - -[SECTION] -TITLE: Plugins - -[REQUIREMENT] -UID: DP-PLU-001 -TITLE: Can send to AWS Kinesis Data Streams through Fluent Bit -STATEMENT: The plugin must be able to send data to an AWS Kinesis Data Stream using Fluent Bit's AWS Kinesis Data Streams output plugin. - -[REQUIREMENT] -UID: DP-PLU-002 -TITLE: Can send to AWS S3 through Fluent Bit -STATEMENT: The plugin must be able to send data to an AWS S3 bucket using Fluent Bit's AWS S3 output plugin. - -[REQUIREMENT] -UID: DP-PLU-003 -TITLE: Can save in a file through Fluent Bit -STATEMENT: The plugin must be able to save data to a file using Fluent Bit's file output plugin. - -[REQUIREMENT] -UID: DP-PLU-004 -TITLE: Can save files metrics through Fluent Bit -STATEMENT: The plugin must be able to save files metrics (i.e. file size, file permissions, etc.) using Fluent Bit's file metrics output plugin. - -[REQUIREMENT] -UID: DP-PLU-005 -TITLE: Can send to HTTP through Fluent Bit -STATEMENT: The plugin must be able to send data to an HTTP server using Fluent Bit's HTTP output plugin. - -[REQUIREMENT] -UID: DP-PLU-006 -TITLE: Can send to MinIO through Fluent Bit -STATEMENT: The plugin must be able to send data to a MinIO server using Fluent Bit's MinIO output plugin. - -[REQUIREMENT] -UID: DP-PLU-007 -TITLE: Can send to PostgreSQL through Fluent Bit -STATEMENT: The plugin must be able to send data to a PostgreSQL database using Fluent Bit's PostgreSQL output plugin. - -[REQUIREMENT] -UID: DP-PLU-008 -TITLE: Can send to Slack through Fluent Bit -STATEMENT: The plugin must be able to send data to a Slack channel using Fluent Bit's Slack output plugin. - -[REQUIREMENT] -UID: DP-PLU-009 -TITLE: Can send to Stdout through Fluent Bit -STATEMENT: The plugin must be able to output data to the console using Fluent Bit's Stdout output plugin. - -[REQUIREMENT] -UID: DP-PLU-010 -TITLE: Can send to RCL stdout -STATEMENT: The plugin must be able to output data to the ROS 2 console using the RCL stdout output plugin. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit File - -[REQUIREMENT] -UID: MP-FLB-FILE-001 -TITLE: Can write data to a file using the Fluent Bit file plugin -STATEMENT: The file plugin should be able to write data to a file in the specified location. The data should be written in the specified format and appended to the file if it already exists. - -[REQUIREMENT] -UID: MP-FLB-FILE-002 -TITLE: Can set the file path -STATEMENT: The file plugin must allow the user to specify the path where the output file will be created. The user should be able to specify an absolute or relative path. - -[REQUIREMENT] -UID: MP-FLB-FILE-003 -TITLE: Can set the file name -STATEMENT: The file plugin must allow the user to specify the name of the output file. The user should be able to specify a file name with or without an extension. - -[REQUIREMENT] -UID: MP-FLB-FILE-004 -TITLE: Can create parent directories if they are not created before -STATEMENT: If the directory where the file is supposed to be saved does not exist, the plugin must be able to create the directory and any required parent directories automatically. The user should not be required to manually create the directory before starting the plugin. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit File metrics - -[REQUIREMENT] -UID: MP-FLB-FILE-M-001 -TITLE: Can track file metadata -STATEMENT: The plugin must be able to track metadata for all files that are sent to their destinations, including the file type, size, and timestamp. - -[REQUIREMENT] -UID: MP-FLB-FILE-M-002 -TITLE: Can delete local files -STATEMENT: The plugin must be able to delete local files that have been sent to all their storage destinations, in order to free up disk space. - -[REQUIREMENT] -UID: MP-FLB-FILE-M-003 -TITLE: Uses custom Fluent Bit plugin -STATEMENT: The plugin must use a custom Fluent Bit plugin located in the fluent_bit_plugins package to track file metadata and manage file deletion. - -[REQUIREMENT] -UID: MP-FLB-FILE-M-004 -TITLE: Supports multiple storage destinations -STATEMENT: The plugin must support tracking files that are sent to multiple storage destinations, such as AWS S3, MinIO, etc. - -[REQUIREMENT] -UID: MP-FLB-FILE-M-005 -TITLE: Stores metadata in PostgreSQL database -STATEMENT: The plugin must store file metadata in a PostgreSQL database, including the file name, status, and the destinations it has been sent to. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit HTTP - -[REQUIREMENT] -UID: MP-FLB-HTTP-001 -TITLE: Supports sending records in JSON format -STATEMENT: The HTTP output plugin should allow the user to specify sending records in JSON format through the 'format' parameter. It must be possible to send multiple headers through the 'header' parameter. - -[REQUIREMENT] -UID: MP-FLB-HTTP-002 -TITLE: Supports HTTP Basic Authentication -STATEMENT: The HTTP output plugin should allow the user to send HTTP Basic Authentication credentials through the 'http_user' and 'http_passwd' parameters. - -[REQUIREMENT] -UID: MP-FLB-HTTP-003 -TITLE: Supports setting HTTP host and port -STATEMENT: The HTTP output plugin should allow the user to set the IP address or hostname of the target HTTP Server and TCP port through the 'host' and 'port' parameters, respectively. - -[REQUIREMENT] -UID: MP-FLB-HTTP-004 -TITLE: Supports setting HTTP URI -STATEMENT: The HTTP output plugin should allow the user to set an optional HTTP URI for the target web server through the 'uri' parameter. - -[REQUIREMENT] -UID: MP-FLB-HTTP-005 -TITLE: Supports logging the response payload -STATEMENT: The HTTP output plugin should allow the user to specify whether or not to log the response payload through the 'log_response_payload' parameter. - -[REQUIREMENT] -UID: MP-FLB-HTTP-006 -TITLE: Supports different date formats -STATEMENT: The system should allow users to specify different date formats for parsing and formatting dates, such as ISO 8601, RFC 3339, Unix timestamp, and custom formats. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit AWS Kinesis Data Streams - -[REQUIREMENT] -UID: MP-FLB-KDS-001 -TITLE: Ingests logs into Amazon Kinesis Data Streams -STATEMENT: The system should be able to ingest logs into Amazon Kinesis Data Streams using the provided Amazon Kinesis Data Streams output plugin. The plugin should support the configuration of the AWS region, the name of the Kinesis Streams Delivery stream to send the logs to, and an optional IAM role ARN for cross-account access. - -[REQUIREMENT] -UID: MP-FLB-KDS-002 -TITLE: Supports custom endpoint for Kinesis API -STATEMENT: The Amazon Kinesis Data Streams output plugin should support the configuration of a custom endpoint for the Kinesis API, in case a user wants to send logs to a custom endpoint instead of the default AWS endpoint. - -[REQUIREMENT] -UID: MP-FLB-KDS-003 -TITLE: Allows customization of log record sent to Kinesis -STATEMENT: The Amazon Kinesis Data Streams output plugin should allow customization of the log record that is sent to Kinesis. It should support the configuration of a key name to send only the value of that key to Kinesis, and the addition of a timestamp to the record under a specified key name, using a strftime compliant format string for the timestamp. - -[REQUIREMENT] -UID: MP-FLB-KDS-004 -TITLE: Handles network errors gracefully -STATEMENT: The Amazon Kinesis Data Streams output plugin should handle network errors gracefully and provide clear error messages to users, indicating the cause of the error. It should also support immediate retry of failed requests to AWS services once, without affecting the normal Fluent Bit retry mechanism with backoff. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit MinIO - -[REQUIREMENT] -UID: MP-FLB-MINIO-001 -TITLE: Can activate secure access to Minio -STATEMENT: The Minio plugin can provide secure access to the Minio server by supporting HTTPS protocol and allowing the use of access keys and secret access keys for authentication. - -[REQUIREMENT] -UID: MP-FLB-MINIO-002 -TITLE: Configurable endpoint for Minio -STATEMENT: The Minio plugin must allow the configuration of the endpoint for the Minio server, including the hostname, port, and scheme. - -[REQUIREMENT] -UID: MP-FLB-MINIO-003 -TITLE: Support for access keys and secret access keys -STATEMENT: The Minio plugin must allow the configuration of access keys and secret access keys to provide authentication to the Minio server. - -[REQUIREMENT] -UID: MP-FLB-MINIO-004 -TITLE: Support for SSL/TLS encryption -STATEMENT: The Minio plugin must support SSL/TLS encryption to ensure secure communication with the Minio server. - -[REQUIREMENT] -UID: MP-FLB-MINIO-005 -TITLE: Configurable bucket creation -STATEMENT: The Minio plugin must allow the configuration of whether a bucket should be created automatically if it does not already exist. - -[REQUIREMENT] -UID: MP-FLB-MINIO-006 -TITLE: Configurable bucket name -STATEMENT: The Minio plugin must allow the configuration of the bucket name to which files will be uploaded. - -[REQUIREMENT] -UID: MP-FLB-MINIO-007 -TITLE: Configurable upload and source fields -STATEMENT: The Minio plugin must allow the configuration of the fields containing the remote and local paths for file uploads. These fields should be separated by dots. - -[/SECTION] - -[SECTION] -TITLE: PostgreSQL - -[REQUIREMENT] -UID: MP-FLB-PSQL-001 -TITLE: Supports PostgreSQL 10 and above -STATEMENT: The plugin should be able to connect and write data to PostgreSQL 10 and above. - -[REQUIREMENT] -UID: MP-FLB-PSQL-002 -TITLE: Handles database connection errors gracefully -STATEMENT: The plugin should handle errors when connecting to the PostgreSQL database gracefully and provide clear error messages to users. - -[REQUIREMENT] -UID: MP-FLB-PSQL-003 -TITLE: Handles SQL query errors gracefully -STATEMENT: The plugin should handle errors when executing SQL queries gracefully and provide clear error messages to users. - -[REQUIREMENT] -UID: MP-FLB-PSQL-004 -TITLE: Supports both async and sync connections -STATEMENT: The plugin should support both async and sync connections to the PostgreSQL database. - -[REQUIREMENT] -UID: MP-FLB-PSQL-005 -TITLE: Allows customization of connection pool size -STATEMENT: The plugin should allow users to customize the minimum and maximum number of connections in the connection pool. - -[REQUIREMENT] -UID: MP-FLB-PSQL-006 -TITLE: Allows specification of table to store data in -STATEMENT: The plugin should allow users to specify the name of the table in the PostgreSQL database where data should be stored. - -[REQUIREMENT] -UID: MP-FLB-PSQL-007 -TITLE: Allows specification of timestamp field -STATEMENT: The plugin should allow users to specify which field in the incoming data contains the timestamp for the record. - -[REQUIREMENT] -UID: MP-FLB-PSQL-008 -TITLE: Supports CockroachDB -STATEMENT: The plugin should be able to connect and write data to CockroachDB when the "cockroachdb" option is set to true. - -[REQUIREMENT] -UID: MP-FLB-PSQL-009 -TITLE: Can set a timeout for remote connection to PostgreSQL -STATEMENT: The system should be able to set a timeout for remote connection to PostgreSQL and terminate them if no data has been received within the timeout. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit S3 - -[REQUIREMENT] -UID: MP-FLB-S3-001 -TITLE: Allows specifying the AWS region of the S3 bucket -STATEMENT: The plugin should allow users to specify the AWS region where the S3 bucket resides. - -[REQUIREMENT] -UID: MP-FLB-S3-002 -TITLE: Allows specifying the S3 bucket name -STATEMENT: The plugin should allow users to specify the name of the S3 bucket where data should be stored. - -[REQUIREMENT] -UID: MP-FLB-S3-003 -TITLE: Allows specifying the time key in the output record -STATEMENT: The plugin should allow users to specify the name of the time key in the output record. If the time key is disabled, the plugin should not include any time information in the S3 object. - -[REQUIREMENT] -UID: MP-FLB-S3-004 -TITLE: Allows specifying the format of the date -STATEMENT: The plugin should allow users to specify the format of the date in the output record. The supported formats should include double, epoch, iso8601, and java_sql_timestamp. - -[REQUIREMENT] -UID: MP-FLB-S3-005 -TITLE: Allows specifying the maximum size of files in S3 -STATEMENT: The plugin should allow users to specify the maximum size of files that can be uploaded to S3. The maximum size should be 50GB and the minimum size should be 1MB. - -[REQUIREMENT] -UID: MP-FLB-S3-006 -TITLE: Allows specifying the size of each 'part' for multipart uploads -STATEMENT: The plugin should allow users to specify the size of each 'part' for multipart uploads. The maximum size should be 50MB. - -[REQUIREMENT] -UID: MP-FLB-S3-007 -TITLE: Allows specifying the time interval for completing an upload and creating a new file in S3 -STATEMENT: The plugin should allow users to specify the time interval for completing an upload and creating a new file in S3. If this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. - -[REQUIREMENT] -UID: MP-FLB-S3-008 -TITLE: Allows specifying the directory to locally buffer data before sending -STATEMENT: The plugin should allow users to specify the directory where data should be buffered before being sent to S3. When multipart uploads are used, data will only be buffered until the upload_chunk_size is reached. S3 will also store metadata about in-progress multipart uploads in this directory. - -[REQUIREMENT] -UID: MP-FLB-S3-009 -TITLE: Allows specifying the size limitation for disk usage in S3 -STATEMENT: The plugin should allow users to specify the size limitation for disk usage in S3. The amount of S3 buffers in the store_dir should be limited to this size to limit disk usage. Note: Use store_dir_limit_size instead of storage.total_limit_size, which can be used for other plugins, because S3 has its own buffering system. - -[REQUIREMENT] -UID: MP-FLB-S3-010 -TITLE: Support for multipart upload -STATEMENT: The plugin should support multipart upload to S3, allowing large files to be uploaded in parts for increased reliability and faster upload times. - -[REQUIREMENT] -UID: MP-FLB-S3-011 -TITLE: Can set custom endpoint -STATEMENT: The plugin should support multipart upload to S3, allowing large files to be uploaded in parts for increased reliability and faster upload times. - -[/SECTION] - -[SECTION] -TITLE: Fluent Bit Slack - -[REQUIREMENT] -UID: MP-FLB-SLACK-001 -TITLE: Can set the webhook used by Slack -STATEMENT: The plugin should support using the webhook to know where to send the data. - -[/SECTION] diff --git a/requirements/destination_server.sdoc b/requirements/destination_server.sdoc deleted file mode 100644 index d1eb8f1ca..000000000 --- a/requirements/destination_server.sdoc +++ /dev/null @@ -1,112 +0,0 @@ -[DOCUMENT] -TITLE: Destination server - -[SECTION] -TITLE: Usability - -[REQUIREMENT] -UID: DS-USAB-001 -TITLE: Can load configuration from YAML -STATEMENT: All configuration can be passed as a standard ROS 2 YAML configuration file. - -[/SECTION] - -[SECTION] -TITLE: Plugin Loading - -[REQUIREMENT] -UID: DS-PLU-001 -TITLE: Can load any number of destination plugins -STATEMENT: The Destination server must be able to load any number of destination plugins that conform to a specific interface. The plugins should be loaded dynamically at runtime. - -[REQUIREMENT] -UID: DS-PLU-002 -TITLE: Can handle errors when loading destination plugins -STATEMENT: If the Destination server attempts to load a plugin that does not conform to the specified interface or the plugin file does not exist, it should throw a custom exception with a clear error message. The node should then stop processing. - -[REQUIREMENT] -UID: DS-PLU-003 -TITLE: Extensibility and customizability -STATEMENT: The system should be easily extensible and customizable, allowing users to add new features, plugins, and functionality to meet their specific requirements in the repository. - -[REQUIREMENT] -UID: DS-PLU-004 -TITLE: External extensibility and customizability -STATEMENT: The system should be easily extensible and customizable, allowing users to add new features, plugins, and functionality to meet their specific requirements in their own ROS packages. - -[REQUIREMENT] -UID: DS-PLU-005 -TITLE: Can reload plugins without restarting the node -STATEMENT: The Destination server should be able to reload plugins dynamically without having to stop and restart the node. This will allow for updates and changes to be made to the plugins without interrupting data processing. - -[REQUIREMENT] -UID: DS-PLU-006 -TITLE: Can unload plugins -STATEMENT: The Destination server should be able to unload plugins dynamically. This will allow for plugins to be removed from the system when they are no longer needed. - -[REQUIREMENT] -UID: DS-PLU-007 -TITLE: Can provide information about loaded plugins -STATEMENT: The Destination server should be able to provide information about the plugins that are currently loaded, such as their names, version numbers, description and json schema. - -[/SECTION] - -[SECTION] -TITLE: General - -[REQUIREMENT] -UID: DS-GEN-001 -TITLE: Can batch samples before sending -STATEMENT: The destination system can batch multiple samples together before sending them to remote destinations, reducing network overhead and improving performance. - -[REQUIREMENT] -UID: DS-GEN-002 -TITLE: Can store data locally if remote connection is not available -STATEMENT: If a remote connection is not available, the system should be able to store data locally and send it over the remote connection when it becomes available. - -[REQUIREMENT] -UID: DS-GEN-003 -TITLE: Can store data locally if remote connection is slow or unreliable -STATEMENT: If a remote connection is slow or unreliable, the system should be able to store data locally and send it over the remote connection when it is stable. - -[/SECTION] - -[SECTION] -TITLE: Backend - -[REQUIREMENT] -UID: MS-BAC-001 -TITLE: Can store data using a backend -STATEMENT: The system should be able to store collected data using a backend, such as Fluent Bit or Redis, to handle backpressure and ensure data persistency. - -[REQUIREMENT] -UID: MS-BAC-002 -TITLE: Support multiple backends -STATEMENT: The system should support multiple backends for storing collected data, handling backpressure, and ensuring data persistency. The selection of a backend should be configurable to allow for easy integration with different systems and tools. - -[REQUIREMENT] -UID: MS-BAC-003 -TITLE: Can store data without a backend -STATEMENT: The system should be able to store collected data without a backend if needed, for example when testing or in environments where a backend is not available or practical to use. The system should provide a fallback option for data storage in such cases. - -[REQUIREMENT] -UID: MS-BAC-004 -TITLE: Support for Fluent Bit -STATEMENT: The system should support the use of Fluent Bit as a backend to handle backpressure and ensure data persistency. - -[REQUIREMENT] -UID: MS-BAC-005 -TITLE: Support for Redis -STATEMENT: The system should be able to use Redis as a backend for storing and retrieving collected data. The system should be able to configure the connection to Redis, including the host, port, and authentication credentials, if necessary. The system should support Redis commands for data manipulation and should handle errors and exceptions gracefully. - -[/SECTION] - -[SECTION] -TITLE: Performance - -[REQUIREMENT] -UID: DS-PERF-001 -TITLE: Supports high throughput and low latency -STATEMENT: The system must support high throughput and low latency to handle large volumes of data and deliver near-real-time insights. - -[/SECTION] diff --git a/tools/e2e/Containerfile b/tools/e2e/Containerfile index 88204b728..37e501e30 100644 --- a/tools/e2e/Containerfile +++ b/tools/e2e/Containerfile @@ -41,8 +41,8 @@ COPY . src/ros2_data_collection # dc_demos/dc_simulation (warehouse-sim demo packages) still target Gazebo Classic # (gazebo_ros / aws_robomaker_small_warehouse_world), which Jazzy dropped and this apt # mirror doesn't carry — COLCON_IGNORE excludes them from both rosdep's scan and -# colcon's build (same mechanism #242 uses for dc_destinations/fluent_bit_*). Porting -# them to gz-sim is tracked separately (#268), unrelated to the zero-loss pipeline here. +# colcon's build. Porting them to gz-sim is tracked separately (#268), unrelated to the +# zero-loss pipeline here. # # rosdep/dc.yaml is registered as a local rosdep source so `rosdep install` resolves # tomlplusplus / msgpack-cxx (header-only C++ libs dc_bridge uses that upstream