Initial working sample (#3)

Author: jkurdek

.gitignore

@@ -35,3 +35,4 @@
 build/
 .cache/
 .vscode/
+samples/

CMakePresets.json

@@ -26,6 +26,7 @@
       "description": "Preset for ARM macOS",
       "generator": "Ninja",
       "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
         "CMAKE_SYSTEM_NAME": "Darwin",
         "VCPKG_TARGET_TRIPLET": "arm64-osx"
       }

include/sparser.h

@@ -21,6 +21,12 @@ constexpr size_t kMaxPred = 10;
 constexpr size_t kMaxConj = 10;
 constexpr size_t kTotalMaxRfs = kMaxRfsInPred * kMaxPred * kMaxConj;
 
+class InputReader {
+   public:
+    static std::string ReadFile(const std::string& filename);
+    static std::vector<std::string_view> ReadRecords(const std::string& input);
+};
+
 struct EstimationResult {
     double total_parser_runtime;
     std::array<double, kTotalMaxRfs> total_rf_runtimes;
@@ -32,10 +38,6 @@ struct RawFilterData {
     std::array<std::array<size_t, kMaxPred>, kMaxConj> rf_count = {};
     std::array<size_t, kMaxConj> pred_count = {};
     size_t conj_count = 0;
-
-    static size_t GetFlatIdx(size_t conj_idx, size_t pred_idx, size_t rf_idx) {
-        return conj_idx * kMaxPred * kMaxRfsInPred + pred_idx * kMaxRfsInPred + rf_idx;
-    }
 };
 
 class RawFilterQueryGenerator {
@@ -44,18 +46,6 @@ class RawFilterQueryGenerator {
     static std::vector<std::string_view> GenerateRawFiltersFromPredicate(const std::string_view& input);
 };
 
-class Sparser {
-   public:
-    explicit Sparser(std::unique_ptr<JsonQueryDriver>&& json_query_driver = {})
-        : json_query_driver_(std::move(json_query_driver)) {}
-
-    EstimationResult Calibrate(const std::vector<std::string_view>& input, const JsonQuery& json_query,
-                               const RawFilterData& rf_data);
-
-   private:
-    std::unique_ptr<JsonQueryDriver> json_query_driver_;
-};
-
 enum class NodeType { INTER, FAIL, PARSE };
 
 struct Node {
@@ -76,6 +66,23 @@ struct Node {
           type(node_type) {}
 };
 
+class Sparser {
+   public:
+    explicit Sparser(std::unique_ptr<JsonQueryDriver>&& json_query_driver = {})
+        : json_query_driver_(std::move(json_query_driver)) {}
+
+    void Run(const std::string& input_path, const JsonQuery& json_query);
+
+    EstimationResult Calibrate(const std::vector<std::string_view>& input, const JsonQuery& json_query,
+                               const RawFilterData& rf_data);
+    void SearchCascade(const std::vector<std::string_view>& input, const JsonQuery& json_query,
+                       const RawFilterData& rf_data, const std::shared_ptr<Node>);
+    void SearchNaive(const std::vector<std::string_view>& input, const JsonQuery& json_query);
+
+   private:
+    std::unique_ptr<JsonQueryDriver> json_query_driver_;
+};
+
 class CascadeBuilder {
    public:
     CascadeBuilder(const PredicateDisjunction& disjunction, const RawFilterData& raw_filter_data)
@@ -88,8 +95,8 @@ class CascadeBuilder {
     std::shared_ptr<Node> parse_node = std::make_shared<Node>(0, 0, 0, nullptr, nullptr, NodeType::PARSE);
     const PredicateDisjunction& disjunction_;
     const RawFilterData& rf_data_;
-    std::bitset<kMaxDepth> used_conjunctions_;
-    std::array<std::array<std::bitset<10>, kMaxDepth>, kMaxDepth> used_predicates_;  // TODO: Add correct dimensions
+    std::bitset<kMaxConj> used_conjunctions_;
+    std::array<std::array<std::bitset<kMaxRfsInPred>, kMaxPred>, kMaxConj> used_rfs_;
 
     std::vector<std::shared_ptr<Node>> HandleFail(const size_t current_depth);
     std::vector<std::shared_ptr<Node>> HandleSuccess(const size_t current_depth, const size_t conjunction_idx);
@@ -112,4 +119,8 @@ class CascadeEvaluator {
     void EvaluateNodeRec(std::shared_ptr<Node> node, std::bitset<kSampleSize> cumulative_bitset);
 };
 
+static inline size_t GetFlatIdx(size_t conj_idx, size_t pred_idx, size_t rf_idx) {
+    return conj_idx * kMaxPred * kMaxRfsInPred + pred_idx * kMaxRfsInPred + rf_idx;
+}
+
 #endif  // SPARSER_H_

scripts/generate_sample_data.py

@@ -0,0 +1,76 @@
+import json
+import random
+from typing import List, Dict, Any
+
+
+def generate_json_records(
+    num_records: int, schema: Dict[str, Any], key: str, value: Any, percentage: float
+) -> List[str]:
+    """
+    Generate newline-delimited JSON records.
+
+    :param num_records: Number of JSON records to generate.
+    :param schema: A dictionary defining the schema for JSON records.
+    :param key: The key for which a specific value will be used.
+    :param value: The value to use for the specified key.
+    :param percentage: Percentage of records that will have the specified value for the key.
+    :return: A list of JSON strings (one per line).
+    """
+    records = []
+    for _ in range(num_records):
+        record = {}
+        for field, field_type in schema.items():
+            if field == key and random.random() < percentage / 100:
+                record[field] = value
+            else:
+                record[field] = generate_value(field_type)
+        records.append(json.dumps(record))
+    return records
+
+
+def generate_value(field_type: Any) -> Any:
+    """
+    Generate a random value based on the field type.
+
+    :param field_type: Type of the field, can be str, int, float, list, or dict.
+    :return: A randomly generated value.
+    """
+    if field_type == str:
+        return random.choice(["alpha", "beta", "gamma", "delta"])
+    elif field_type == int:
+        return random.randint(0, 100)
+    elif field_type == float:
+        return round(random.uniform(0, 100), 2)
+    elif field_type == list:
+        return random.sample(["a", "b", "c", "d", "e"], k=random.randint(1, 3))
+    elif field_type == dict:
+        return {
+            "subfield1": random.choice(["x", "y", "z"]),
+            "subfield2": random.randint(1, 10),
+        }
+    else:
+        return None
+
+
+# Example usage
+if __name__ == "__main__":
+    schema_definition = {
+        "id": int,
+        "name": str,
+        "score": float,
+        "tags": list,
+        "details": dict,
+    }
+
+    json_records = generate_json_records(
+        num_records=100000,
+        schema=schema_definition,
+        key="name",
+        value="Trump",
+        percentage=30,
+    )
+
+    with open("samples/generated_records.json", "w") as file:
+        file.write("\n".join(json_records))
+
+    print("JSON records have been written to 'samples/generated_records.json'")

src/CMakeLists.txt

@@ -2,3 +2,4 @@ add_subdirectory(sparser)
 
 add_executable(SparserMain main.cpp)
 target_link_libraries(SparserMain PRIVATE SparserCpp)
+target_link_libraries(SparserMain PRIVATE rapidjson)

src/main.cpp

@@ -1,37 +1,9 @@
-#include <exception>
-#include <fstream>
-#include <iostream>
 #include <span>
-#include <stdexcept>
-#include <string>
-#include <string_view>
-
-constexpr double GIGABYTE = 1e9;
-
-/**
- * Reads the contents of a file into a dynamically allocated buffer.
- *
- * @param filename The name of the file to be read.
- * @return A string containing the contents of the file.
- */
-std::string readFile(std::string filename) {
-    std::ifstream file(filename, std::ios::binary | std::ios::ate);
-
-    if (!file) {
-        throw std::runtime_error("Error opening file: " + std::string(filename));
-    }
-
-    auto fileSize = file.tellg();
-    file.seekg(0, std::ios::beg);
 
-    std::string buffer(fileSize, '\0');
-
-    if (!file.read(buffer.data(), fileSize)) {
-        throw std::runtime_error("Error reading file: " + std::string(filename));
-    }
+#include "json_facade.h"
+#include "sparser.h"
 
-    return buffer;
-}
+// constexpr double GIGABYTE = 1e9;
 
 int main(int argc, char* argv[]) {
     try {
@@ -44,13 +16,14 @@ int main(int argc, char* argv[]) {
 
         const std::string filename = args[1];
 
-        std::cout << "Reading file: " << filename << "\n";
-        auto buffer = readFile(filename);
-        std::cout << "Done reading! File size: " << static_cast<double>(buffer.size()) / GIGABYTE << " GB" << "\n";
+        Predicate pred1{.key = "name", .value = "Trump"};
 
-        if (buffer.empty()) {
-            return 1;
-        }
+        PredicateConjunction conj1{{pred1}};
+        PredicateDisjunction disj{{conj1}};
+
+        auto json_query_driver = new JsonQueryDriver(std::make_unique<RapidJsonFacade>());
+        auto sparser = Sparser(std::unique_ptr<JsonQueryDriver>(json_query_driver));
+        sparser.Run(filename, JsonQuery(disj));
 
     } catch (const std::exception& e) {
         std::cerr << "Exception caught: " << e.what() << "\n";

src/sparser/json_facade.cpp

@@ -37,7 +37,7 @@ std::ostream& operator<<(std::ostream& os, const JsonQuery& query) {
 }
 
 void RapidJsonFacade::Parse(std::string_view jsonStr) {
-    rapidjson::ParseResult ok = doc_.Parse(jsonStr.data());
+    rapidjson::ParseResult ok = doc_.Parse(jsonStr.data(), jsonStr.size());
     if (!ok || !doc_.IsObject()) {
         throw std::runtime_error("Failed to parse JSON string");
     }