Merge branch 'gh-343'

Author: rizsotto

.github/workflows/build_on_push.yml

@@ -23,12 +23,12 @@ jobs:
         if:  startsWith( matrix.os, 'ubuntu' )
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y cmake pkg-config gfortran nvidia-cuda-toolkit valgrind
+          sudo apt-get install -y cmake pkg-config gfortran nvidia-cuda-toolkit valgrind libsqlite3-dev
 
       - name: Install package dependencies
         if:  startsWith( matrix.os, 'macos' )
         run: |
-          brew install cmake pkg-config fmt spdlog nlohmann-json grpc
+          brew install cmake pkg-config fmt spdlog nlohmann-json grpc sqlite
           echo "PKG_CONFIG_PATH=$(brew --prefix)/opt/openssl@1.1/lib/pkgconfig" >> $GITHUB_ENV
 
       - name: Set up Python

.github/workflows/codeql-analysis.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y cmake pkg-config
+          sudo apt-get install -y cmake pkg-config libsqlite3-dev
 
       - name: Prepare
         run: |

INSTALL.md

@@ -23,6 +23,7 @@ and build locally.
 - [fmt](https://github.com/fmtlib/fmt) >= 6.2
 - [spdlog](https://github.com/gabime/spdlog) >= 1.5
 - [json](https://github.com/nlohmann/json) >= 3.7
+- [sqlite](https://www.sqlite.org/) >= 3.14
 
 Developer dependencies:
 
@@ -57,29 +58,30 @@ it before.
 Install dependencies from packages on Fedora 32/33
 
     dnf install python cmake pkg-config
-    dnf install json-devel spdlog-devel fmt-devel grpc-devel grpc-plugins
+    dnf install json-devel spdlog-devel fmt-devel sqlite-devel grpc-devel grpc-plugins
     dnf install gtest-devel gmock-devel # optional for running the tests
 
 Install dependencies from packages on Arch
 
     pacman -S python cmake pkg-config
-    pacman -S grpc spdlog fmt nlohmann-json
+    pacman -S grpc spdlog fmt nlohmann-json sqlite
     pacman -S gtest gmock # optional for running the tests
 
 Install dependencies from packages on Ubuntu 20.04
 
     apt-get install python cmake pkg-config
-    apt-get install libfmt-dev libspdlog-dev nlohmann-json3-dev
+    apt-get install libfmt-dev libspdlog-dev nlohmann-json3-dev libsqlite3-dev
     apt-get install libgrpc++-dev protobuf-compiler-grpc libssl-dev
 
 Install dependencies from packages from Brew
 
-    brew install fmt spdlog nlohmann-json grpc pkg-config
+    brew install fmt spdlog nlohmann-json sqlite grpc pkg-config
 
 Install dependencies from packages on Alpine edge
 
     apk add git cmake pkgconf make g++
-    apk add protobuf-dev grpc-dev fmt-dev spdlog-dev nlohmann-json c-ares-dev
+    apk add fmt-dev spdlog-dev nlohmann-json sqlite-dev
+    apk add protobuf-dev grpc-dev c-ares-dev
 
 ### Platform: macOS
 

source/CMakeLists.txt

@@ -28,6 +28,7 @@ find_package(fmt REQUIRED)
 find_package(spdlog REQUIRED)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(gRPC REQUIRED IMPORTED_TARGET protobuf grpc++)
+pkg_check_modules(sqlite REQUIRED IMPORTED_TARGET sqlite3)
 
 if (UNIX AND NOT APPLE)
     set(SUPPORT_PRELOAD 1)

source/bear/main.cc

@@ -201,9 +201,9 @@ namespace {
         {
             auto commands = args.as_string(OUTPUT)
                     .map<fs::path>([](const auto &output) {
-                        return fs::path(output).replace_extension(".commands.json");
+                        return fs::path(output).replace_extension(".sqlite3");
                     })
-                    .unwrap_or(fs::path("commands.json"));
+                    .unwrap_or(fs::path("commands.sqlite3"));
 
             auto environment = sys::env::from(const_cast<const char **>(envp));
             auto intercept = prepare_intercept(args, environment, commands);

source/citnames/CMakeLists.txt

@@ -2,6 +2,7 @@
 add_library(citnames_json_a OBJECT
         source/Configuration.cc
         source/Output.cc
+        source/semantic/Command.cc
         )
 
 target_link_libraries(citnames_json_a PUBLIC result_a)
@@ -49,6 +50,7 @@ target_link_libraries(citnames report_a)
 target_link_libraries(citnames flags_a)
 target_link_libraries(citnames shell_a)
 target_link_libraries(citnames sys_a)
+target_link_libraries(citnames rpc_a)
 target_link_libraries(citnames fmt::fmt)
 target_link_libraries(citnames spdlog::spdlog)
 target_link_libraries(citnames nlohmann_json::nlohmann_json)
@@ -84,6 +86,7 @@ if (ENABLE_UNIT_TESTS)
     target_link_libraries(citnames_unit_test flags_a)
     target_link_libraries(citnames_unit_test shell_a)
     target_link_libraries(citnames_unit_test sys_a)
+    target_link_libraries(citnames_unit_test rpc_a)
     target_link_libraries(citnames_unit_test fmt::fmt)
     target_link_libraries(citnames_unit_test nlohmann_json::nlohmann_json)
     target_link_libraries(citnames_unit_test PkgConfig::GTest ${CMAKE_THREAD_LIBS_INIT})

source/citnames/source/Application.cc

@@ -23,7 +23,6 @@
 #include "Configuration.h"
 #include "Output.h"
 #include "semantic/Tool.h"
-#include "intercept/output/Report.h"
 
 #include <filesystem>
 
@@ -118,29 +117,30 @@ namespace {
 namespace cs {
 
     rust::Result<int> Command::execute() const {
+        cs::CompilationDatabase output(configuration_.output.format, configuration_.output.content);
+
         // get current compilations from the input.
-        return report_serializer_.from_json(arguments_.input)
+        return ic::EventsDatabase::open(arguments_.input)
                 .map<Entries>([this](const auto &commands) {
-                    spdlog::debug("commands have read. [size: {}]", commands.executions.size());
                     auto compilations = tools_.transform(commands);
                     // remove duplicates
                     return merge({}, compilations);
                 })
-                .and_then<Entries>([this](const auto &compilations) {
+                .and_then<Entries>([this, &output](const auto &compilations) {
                     // read back the current content and extend with the new elements.
                     spdlog::debug("compilation entries created. [size: {}]", compilations.size());
                     return (arguments_.append)
-                           ? output_.from_json(arguments_.output.c_str())
+                           ? output.from_json(arguments_.output.c_str())
                                    .template map<Entries>([&compilations](auto old_entries) {
                                        spdlog::debug("compilation entries have read. [size: {}]", old_entries.size());
                                        return merge(compilations, old_entries);
                                    })
                            : rust::Result<Entries>(rust::Ok(compilations));
                 })
-                .and_then<size_t>([this](const auto &compilations) {
+                .and_then<size_t>([this, &output](const auto &compilations) {
                     // write the entries into the output file.
                     spdlog::debug("compilation entries to output. [size: {}]", compilations.size());
-                    return output_.to_json(arguments_.output.c_str(), compilations);
+                    return output.to_json(arguments_.output.c_str(), compilations);
                 })
                 .map<int>([](auto size) {
                     // just map to success exit code if it was successful.
@@ -159,7 +159,7 @@ namespace cs {
                 "citnames",
                 VERSION,
                 {
-                        {cs::INPUT,      {1, false, "path of the input file",                    {"commands.json"},         std::nullopt}},
+                        {cs::INPUT,      {1, false, "path of the input file",                    {"commands.sqlite3"},         std::nullopt}},
                         {cs::OUTPUT,     {1, false, "path of the result file",                   {"compile_commands.json"}, std::nullopt}},
                         {cs::CONFIG,     {1, false, "path of the config file",                   std::nullopt,              std::nullopt}},
                         {cs::APPEND,     {0, false, "append to output, instead of overwrite it", std::nullopt,              std::nullopt}},
@@ -173,17 +173,16 @@ namespace cs {
 
         auto arguments = into_arguments(args);
         auto configuration = into_configuration(args, environment);
-        auto semantic = configuration.and_then<cs::semantic::Tools>([](auto config) {
-            return semantic::Tools::from(config.compilation);
-        });
+        auto tools = configuration
+                .and_then<cs::semantic::Tools>([](auto config) {
+                    return semantic::Tools::from(config.compilation);
+                });
 
-        return rust::merge(arguments, configuration, semantic)
+        return rust::merge(arguments, configuration, tools)
                 .map<ps::CommandPtr>([](auto tuples) {
-                    const auto&[arguments, configuration, semantic] = tuples;
+                    const auto&[arguments, configuration, tools] = tuples;
                     // read the configuration
-                    cs::CompilationDatabase output(configuration.output.format, configuration.output.content);
-                    report::ReportSerializer report_serializer;
-                    return std::make_unique<Command>(arguments, report_serializer, semantic, output);
+                    return std::make_unique<Command>(arguments, configuration, tools);
                 });
     }
 }

source/citnames/source/Application.h

@@ -21,12 +21,13 @@
 
 #include "Output.h"
 #include "semantic/Tool.h"
+#include "intercept/EventsDatabase.h"
 #include "libmain/ApplicationFromArgs.h"
 #include "libresult/Result.h"
 #include "libsys/Environment.h"
-#include "intercept/output/Report.h"
 
 #include <filesystem>
+#include <utility>
 
 namespace fs = std::filesystem;
 
@@ -39,24 +40,20 @@ namespace cs {
     };
 
     struct Command : ps::Command {
-        Command(Arguments arguments,
-                report::ReportSerializer report_serializer,
-                cs::semantic::Tools tools,
-                cs::CompilationDatabase output)
+
+        Command(Arguments arguments, cs::Configuration configuration, cs::semantic::Tools tools)
                 : ps::Command()
-                , arguments_(arguments)
-                , report_serializer_(report_serializer)
+                , arguments_(std::move(arguments))
+                , configuration_(std::move(configuration))
                 , tools_(tools)
-                , output_(output)
         { }
 
         [[nodiscard]] rust::Result<int> execute() const override;
 
     private:
         Arguments arguments_;
-        report::ReportSerializer report_serializer_;
+        cs::Configuration configuration_;
         cs::semantic::Tools tools_;
-        cs::CompilationDatabase output_;
     };
 
     struct Application : ps::ApplicationFromArgs {

source/citnames/source/semantic/Command.cc

@@ -0,0 +1,46 @@
+/*  Copyright (C) 2012-2020 by László Nagy
+    This file is part of Bear.
+
+    Bear is a tool to generate compilation database for clang tooling.
+
+    Bear is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Bear is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Command.h"
+
+#include <iostream>
+
+#include <nlohmann/json.hpp>
+
+namespace cs::semantic {
+
+    bool operator==(const Command& lhs, const Command& rhs)
+    {
+        return (lhs.program == rhs.program)
+               && (lhs.arguments == rhs.arguments)
+               && (lhs.working_dir == rhs.working_dir)
+               && (lhs.environment == rhs.environment);
+    }
+
+    std::ostream& operator<<(std::ostream& os, const Command& rhs)
+    {
+        nlohmann::json payload = nlohmann::json {
+                { "program", rhs.program },
+                { "arguments", nlohmann::json(rhs.arguments) },
+                { "working_dir", rhs.working_dir },
+        };
+        os << payload;
+        return os;
+    }
+}

source/citnames/source/semantic/Command.h

@@ -0,0 +1,44 @@
+/*  Copyright (C) 2012-2020 by László Nagy
+    This file is part of Bear.
+
+    Bear is a tool to generate compilation database for clang tooling.
+
+    Bear is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Bear is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <list>
+#include <map>
+#include <string>
+#include <iosfwd>
+
+namespace fs = std::filesystem;
+
+namespace cs::semantic {
+
+    // This represents the executed command itself. Describes all the
+    // context that the caller was given. And these are the those parameters
+    // which are essential for re-run the command.
+    struct Command {
+        fs::path program;
+        std::list<std::string> arguments;
+        fs::path working_dir;
+        std::map<std::string, std::string> environment;
+    };
+
+    bool operator==(const Command& lhs, const Command& rhs);
+    std::ostream& operator<<(std::ostream&, const Command&);
+}