Merge pull request #4 from Irval1337/added-csv-interface

Add csv interface

Author: Irval1337

.gitignore

@@ -42,4 +42,5 @@
 
 build/
 .vscode/
-.cache/
+.cache/
+.clangd

CMakeLists.txt

@@ -12,10 +12,10 @@ option(COLUMNAR_BUILD_BENCHMARKS "Build benchmarks" ON)
 include(cmake/deps.cmake)
 
 add_library(columnar_lib
-  src/add.cpp
+  src/core/column.cpp
 )
 
-target_include_directories(columnar_lib PUBLIC include)
+target_include_directories(columnar_lib PUBLIC ${PROJECT_SOURCE_DIR}/include)
 
 target_compile_options(columnar_lib PRIVATE -Wall -Wextra -Wpedantic)
 

apps/converter/main.cpp

@@ -3,15 +3,13 @@
 
 #include <iostream>
 
-#include "columnar/add.h"
-
-ABSL_FLAG(int, x, 1, "x");
-ABSL_FLAG(int, y, 2, "y");
+// ABSL_FLAG(int, x, 1, "x");
+// ABSL_FLAG(int, y, 2, "y");
 
 int main(int argc, char** argv) {
     absl::ParseCommandLine(argc, argv);
-    int x = absl::GetFlag(FLAGS_x);
-    int y = absl::GetFlag(FLAGS_y);
-    std::cout << "Add(" << x << ", " << y << ") = " << columnar::Add(x, y) << "\n";
+    // int x = absl::GetFlag(FLAGS_x);
+    // int y = absl::GetFlag(FLAGS_y);
+    std::cout << "Hello" << "\n";
     return 0;
 }

benchmarks/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_executable(columnar_bench
-  bench_bruh.cpp
+  bench_csv.cpp
 )
 
 target_link_libraries(columnar_bench

benchmarks/bench_bruh.cpp

@@ -0,0 +1,69 @@
+#include <benchmark/benchmark.h>
+#include <csv/csv.h>
+#include <core/schema.h>
+
+#include <sstream>
+
+using namespace columnar;  // NOLINT
+
+std::string GenerateCSV(std::size_t rows) {
+    std::ostringstream ss;
+    for (std::size_t i = 0; i < rows; ++i) {
+        ss << i << ",value" << i << ",0.123\n";
+    }
+    return ss.str();
+}
+
+void BenchCSVRowReader(benchmark::State& state) {
+    std::string data = GenerateCSV(state.range(0));
+    for (auto s : state) {
+        std::istringstream in(data);
+        csv::CSVRowReader reader(in);
+        while (reader.ReadRow()) {
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+BENCHMARK(BenchCSVRowReader)->Range(100, 10000);
+
+void BenchCSVBatchReader(benchmark::State& state) {
+    core::Schema schema({core::Field("id", core::DataType::Int64),
+                         core::Field("name", core::DataType::String),
+                         core::Field("value", core::DataType::Double)});
+
+    std::string data = GenerateCSV(state.range(0));
+    for (auto s : state) {
+        std::istringstream in(data);
+        csv::CSVBatchReader reader(in, schema, {});
+        while (reader.ReadNext()) {
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+BENCHMARK(BenchCSVBatchReader)->Range(100, 10000);
+
+void BenchCSVBatchWriter(benchmark::State& state) {
+    core::Schema schema(
+        {core::Field("id", core::DataType::Int64), core::Field("name", core::DataType::String)});
+
+    core::Batch batch(schema);
+    for (int i = 0; i < state.range(0); ++i) {
+        batch.ColumnAt(0).AppendFromString(std::to_string(i));
+        batch.ColumnAt(1).AppendFromString("hello mir");
+    }
+
+    for (auto s : state) {
+        std::ostringstream out;
+        csv::CSVBatchWriter writer(out, {});
+        writer.Write(batch);
+        writer.Flush();
+        benchmark::DoNotOptimize(out.str());
+    }
+
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+BENCHMARK(BenchCSVBatchWriter)->Range(100, 10000);
+
+BENCHMARK_MAIN();

benchmarks/bench_csv.cpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <core/schema.h>
+#include <core/column.h>
+#include <util/macro.h>
+
+namespace columnar::core {
+class Batch {
+public:
+    Batch() = default;
+
+    Batch(Schema schema) : schema_(std::move(schema)) {
+        columns_.reserve(schema_.FieldsCount());
+        for (auto& field : schema_.GetFields()) {
+            columns_.emplace_back(MakeColumn(field.type, field.nullable));
+        }
+    }
+
+    Batch(Schema schema, std::size_t reserve_rows) : Batch(std::move(schema)) {
+        Reserve(reserve_rows);
+    }
+
+    const Schema& GetSchema() const {
+        return schema_;
+    }
+
+    std::size_t ColumnsCount() const {
+        return columns_.size();
+    }
+
+    std::size_t RowsCount() const {
+        if (columns_.empty()) {
+            return 0;
+        }
+        return columns_[0]->Size();
+    }
+
+    void Reserve(std::size_t n) {
+        for (auto& c : columns_) {
+            c->Reserve(n);
+        }
+    }
+
+    const Column& ColumnAt(std::size_t i) const {
+        return *columns_.at(i);
+    }
+
+    Column& ColumnAt(std::size_t i) {
+        return *columns_.at(i);
+    }
+
+    const std::vector<std::unique_ptr<Column>>& GetColumns() const {
+        return columns_;
+    }
+
+    std::vector<std::unique_ptr<Column>>& GetColumns() {
+        return columns_;
+    }
+
+    void Validate() const {
+        if (columns_.empty()) {
+            return;
+        }
+        for (auto& c : columns_) {
+            if (c->Size() != columns_[0]->Size()) {
+                THROW_RUNTIME_ERROR("Different columns sizes");
+            }
+        }
+    }
+
+private:
+    Schema schema_;
+    std::vector<std::unique_ptr<Column>> columns_;
+};
+}  // namespace columnar::core

include/columnar/add.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <core/batch.h>
+
+#include <optional>
+
+namespace columnar::core {
+class BatchReader {
+public:
+    virtual ~BatchReader() = default;
+
+    virtual std::optional<Batch> ReadNext() = 0;
+
+    virtual const core::Schema& GetSchema() const = 0;
+};
+}  // namespace columnar::core

include/core/batch.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <core/batch.h>
+
+namespace columnar::core {
+class BatchWriter {
+public:
+    virtual ~BatchWriter() = default;
+
+    virtual void Write(const Batch& batch) = 0;
+
+    virtual void Flush() = 0;
+};
+}  // namespace columnar::core