Feature/python new datatypes (#641)

cpp/src/common/allocator/my_string.h

@@ -78,6 +78,21 @@ struct String {
         memcpy(buf_, str.buf_, len_);
         return common::E_OK;
     }
+
+    FORCE_INLINE int dup_from(const char *str, uint32_t len,
+                              common::PageArena &pa) {
+        len_ = len;
+        if (UNLIKELY(len_ == 0)) {
+            return common::E_OK;
+        }
+        buf_ = pa.alloc(len_);
+        if (IS_NULL(buf_)) {
+            return common::E_OOM;
+        }
+        memcpy(buf_, str, len_);
+        return common::E_OK;
+    }
+
     FORCE_INLINE int build_from(const String &s1, const String &s2,
                                 common::PageArena &pa) {
         len_ = s1.len_ + s2.len_;

cpp/src/common/record.h

@@ -53,9 +53,8 @@ struct DataPoint {
         int64_t i64_val_;
         float float_val_;
         double double_val_;
-        common::String *str_val_;
     } u_;
-    TextType text_val_;
+    common::String text_val_;
 
     DataPoint(const std::string &measurement_name, bool b)
         : measurement_name_(measurement_name), text_val_() {
@@ -82,19 +81,12 @@ struct DataPoint {
         u_.double_val_ = d;
     }
 
-    DataPoint(const std::string &measurement_name, common::String &str,
-              common::PageArena &pa)
+    DataPoint(const std::string &measurement_name, common::String str)
         : measurement_name_(measurement_name), text_val_() {
-        char *p_buf = (char *)pa.alloc(sizeof(common::String));
-        u_.str_val_ = new (p_buf) common::String();
-        u_.str_val_->dup_from(str, pa);
+        text_val_.buf_ = str.buf_;
+        text_val_.len_ = str.len_;
     }
 
-    // DataPoint(const std::string &measurement_name, Text &text),
-    //   : measurement_name_(measurement_name),
-    //     data_type_(common::TEXT),
-    //     text_val_(text) {}
-
     DataPoint(const std::string &measurement_name)
         : isnull(true), measurement_name_(measurement_name) {}
     void set_i32(int32_t i32) {
@@ -126,7 +118,7 @@ struct TsRecord {
     }
 
     TsRecord(const std::string &device_name, const int64_t &timestamp)
-        : device_id_(device_name), timestamp_(timestamp) {
+        : timestamp_(timestamp), device_id_(device_name) {
         pa.init(512, common::MOD_TSFILE_READER);
     }
 
@@ -150,7 +142,7 @@ template <>
 inline int TsRecord::add_point(const std::string &measurement_name,
                                common::String val) {
     int ret = common::E_OK;
-    points_.emplace_back(DataPoint(measurement_name, val, pa));
+    points_.emplace_back(DataPoint(measurement_name, val));
     return ret;
 }
 

cpp/src/common/row_record.h

@@ -103,6 +103,7 @@ struct Field {
         switch (type_) {
             case common::TSDataType::BOOLEAN:
                 return value_.bval_;
+            case common::TSDataType::DATE:
             case common::TSDataType::INT32:
                 return value_.ival_;
             case common::TSDataType::TIMESTAMP:

cpp/src/cwrapper/tsfile_cwrapper.cc

@@ -280,6 +280,18 @@ TsRecord _ts_record_new(const char *device_id, Timestamp timestamp,
         return common::E_OK;                                         \
     }
 
+ERRNO _insert_data_into_ts_record_by_name_string_with_len(
+    TsRecord data, const char *measurement_name, const char *value,
+    const uint32_t value_len) {
+    auto *record = (storage::TsRecord *)data;
+    if (record->points_.size() + 1 > record->points_.capacity())
+        return common::E_BUF_NOT_ENOUGH;
+    common::String str_value;
+    str_value.dup_from(value, value_len, record->pa);
+    record->add_point(measurement_name, str_value);
+    return common::E_OK;
+}
+
 INSERT_DATA_INTO_TS_RECORD_BY_NAME_DEF(int32_t);
 INSERT_DATA_INTO_TS_RECORD_BY_NAME_DEF(int64_t);
 INSERT_DATA_INTO_TS_RECORD_BY_NAME_DEF(bool);

cpp/src/cwrapper/tsfile_cwrapper.h

@@ -35,6 +35,8 @@ typedef enum {
     TS_DATATYPE_DOUBLE = 4,
     TS_DATATYPE_TEXT = 5,
     TS_DATATYPE_VECTOR = 6,
+    TS_DATATYPE_DATE = 9,
+    TS_DATATYPE_BLOB = 10,
     TS_DATATYPE_STRING = 11,
     TS_DATATYPE_NULL_TYPE = 254,
     TS_DATATYPE_INVALID = 255
@@ -624,6 +626,10 @@ INSERT_DATA_INTO_TS_RECORD_BY_NAME(bool);
 INSERT_DATA_INTO_TS_RECORD_BY_NAME(float);
 INSERT_DATA_INTO_TS_RECORD_BY_NAME(double);
 
+ERRNO _insert_data_into_ts_record_by_name_string_with_len(
+    TsRecord data, const char* measurement_name, const char* value,
+    const uint32_t value_len);
+
 // Write a tablet into a device.
 ERRNO _tsfile_writer_write_tablet(TsFileWriter writer, Tablet tablet);
 

cpp/src/writer/tsfile_writer.cc

@@ -657,7 +657,7 @@ int TsFileWriter::write_point(ChunkWriter *chunk_writer, int64_t timestamp,
         case common::BLOB:
         case common::TEXT:
         case common::STRING:
-            return chunk_writer->write(timestamp, *point.u_.str_val_);
+            return chunk_writer->write(timestamp, point.text_val_);
         default:
             return E_INVALID_DATA_POINT;
     }
@@ -689,7 +689,7 @@ int TsFileWriter::write_point_aligned(ValueChunkWriter *value_chunk_writer,
         case common::BLOB:
         case common::TEXT:
         case common::STRING:
-            return value_chunk_writer->write(timestamp, point.u_.str_val_,
+            return value_chunk_writer->write(timestamp, point.text_val_,
                                              isnull);
         default:
             return E_INVALID_DATA_POINT;

cpp/test/cwrapper/cwrapper_test.cc

@@ -19,6 +19,17 @@
 #include <gtest/gtest.h>
 #include <unistd.h>
 #include <utils/db_utils.h>
+
+#include "common/row_record.h"
+#include "cwrapper/tsfile_cwrapper.h"
+#include "reader/result_set.h"
+#include "reader/tsfile_reader.h"
+#include "writer/tsfile_writer.h"
+
+namespace storage {
+class TsFileReader;
+}
+
 extern "C" {
 #include "cwrapper/errno_define_c.h"
 #include "cwrapper/tsfile_cwrapper.h"
@@ -28,23 +39,135 @@ extern "C" {
 #include "utils/errno_define.h"
 
 namespace cwrapper {
-class CWrapperTest : public testing::Test {};
-
-// TEST_F(CWrapperTest, RegisterTimeSeries) {
-//     ERRNO code = 0;
-//     char* temperature = strdup("temperature");
-//     TimeseriesSchema ts_schema{temperature, TS_DATATYPE_INT32,
-//                                TS_ENCODING_PLAIN,
-//                                TS_COMPRESSION_UNCOMPRESSED};
-//     remove("cwrapper_register_timeseries.tsfile");
-//     TsFileWriter writer =
-//     tsfile_writer_new("cwrapper_register_timeseries.tsfile", &code);
-//     ASSERT_EQ(code, 0);
-//     code = tsfile_writer_register_timeseries(writer, "device1", &ts_schema);
-//     ASSERT_EQ(code, 0);
-//     free(temperature);
-//     tsfile_writer_close(writer);
-// }
+class CWrapperTest : public testing::Test {
+   public:
+    static void ASSERT_OK(ERRNO code, const char* msg = "") {
+        ASSERT_EQ(code, RET_OK) << msg;
+    }
+};
+
+TEST_F(CWrapperTest, TestForPythonInterfaceInsert) {
+    ERRNO code = 0;
+    const char* filename = "cwrapper_for_python.tsfile";
+    remove(filename);  // Clean up any existing file
+
+    // Device and measurement definitions
+    char* device_id = strdup("root.device1");
+    char* str_measurement_id = strdup("str_measurement");
+    char* text_measurement_id = strdup("text_measurement");
+    char* date_measurement_id = strdup("date_measurement");
+
+    // Define time series schemas for different data types
+    timeseries_schema str_measurement;
+    str_measurement.timeseries_name = str_measurement_id;
+    str_measurement.compression = TS_COMPRESSION_UNCOMPRESSED;
+    str_measurement.data_type = TS_DATATYPE_STRING;
+    str_measurement.encoding = TS_ENCODING_PLAIN;
+
+    timeseries_schema text_measurement;
+    text_measurement.timeseries_name = text_measurement_id;
+    text_measurement.compression = TS_COMPRESSION_UNCOMPRESSED;
+    text_measurement.data_type = TS_DATATYPE_TEXT;
+    text_measurement.encoding = TS_ENCODING_PLAIN;
+
+    timeseries_schema date_measurement;
+    date_measurement.timeseries_name = date_measurement_id;
+    date_measurement.compression = TS_COMPRESSION_UNCOMPRESSED;
+    date_measurement.data_type = TS_DATATYPE_DATE;
+    date_measurement.encoding = TS_ENCODING_PLAIN;
+
+    // Create TsFile writer
+    auto* writer = (storage::TsFileWriter*)_tsfile_writer_new(
+        filename, 128 * 1024 * 1024, &code);
+    ASSERT_OK(code, "create writer failed");
+
+    // Register time series with the writer
+    ASSERT_OK(
+        _tsfile_writer_register_timeseries(writer, device_id, &str_measurement),
+        "register timeseries failed");
+
+    ASSERT_OK(_tsfile_writer_register_timeseries(writer, device_id,
+                                                 &text_measurement),
+              "register timeseries failed");
+
+    ASSERT_OK(_tsfile_writer_register_timeseries(writer, device_id,
+                                                 &date_measurement),
+              "register timeseries failed");
+
+    // Create a new time series record
+    auto* record = (storage::TsRecord*)_ts_record_new(device_id, 0, 3);
+
+    // Insert string data
+    const char* test_str = "test_string";
+    ASSERT_OK(_insert_data_into_ts_record_by_name_string_with_len(
+                  record, str_measurement_id, test_str, strlen(test_str)),
+              "insert data failed");
+
+    // Insert text data
+    const char* test_text = "test_text";
+    ASSERT_OK(_insert_data_into_ts_record_by_name_string_with_len(
+                  record, text_measurement_id, test_text, strlen(test_text)),
+              "insert data failed");
+
+    // Insert date data - NOTE: There's a bug here, should use
+    // date_measurement_id
+    int32_t test_date = 20251118;
+    ASSERT_OK(_insert_data_into_ts_record_by_name_int32_t(
+                  record, date_measurement_id, test_date),
+              "insert data failed");
+
+    // Write the record to file and close writer
+    ASSERT_OK(_tsfile_writer_write_ts_record(writer, record),
+              "write record failed");
+    ASSERT_OK(_tsfile_writer_flush(writer), "flush failed");
+    ASSERT_OK(_tsfile_writer_close(writer), "close writer failed");
+
+    // Create reader to verify the written data
+    auto* reader = (storage::TsFileReader*)tsfile_reader_new(filename, &code);
+    ASSERT_OK(code, "create reader failed");
+
+    // Query the data we just wrote
+    char* sensors[] = {str_measurement_id, text_measurement_id,
+                       date_measurement_id};
+    auto* result = (storage::ResultSet*)_tsfile_reader_query_device(
+        reader, device_id, sensors, 3, 0, 100, &code);
+    ASSERT_OK(code, "query device failed");
+
+    // Verify the retrieved data matches what we inserted
+    bool has_next = false;
+    int row_count = 0;
+    while (result->next(has_next) == common::E_OK && has_next) {
+        // Verify timestamp
+        EXPECT_EQ(result->get_value<int64_t>(1), row_count);
+
+        // Verify string data
+        const common::String* str = result->get_value<common::String*>(2);
+        EXPECT_EQ(strlen(test_str), str->len_);
+        const char* ret_char =
+            tsfile_result_set_get_value_by_index_string(result, 2);
+        EXPECT_EQ(strcmp(test_str, ret_char), 0);
+
+        // Verify text data
+        const common::String* text = result->get_value<common::String*>(3);
+        EXPECT_EQ(strlen(test_text), text->len_);
+        const char* ret_text =
+            tsfile_result_set_get_value_by_index_string(result, 3);
+        EXPECT_EQ(strcmp(test_text, ret_text), 0);
+
+        // Verify date data
+        int32_t ret_date =
+            tsfile_result_set_get_value_by_index_int32_t(result, 4);
+        EXPECT_EQ(test_date, ret_date);
+
+        row_count++;
+    }
+
+    ASSERT_OK(reader->close(), "close reader failed");
+    delete device_id;
+    delete str_measurement_id;
+    delete text_measurement_id;
+    delete date_measurement_id;
+}
 
 TEST_F(CWrapperTest, WriterFlushTabletAndReadData) {
     ERRNO code = 0;

cpp/test/reader/table_view/tsfile_reader_table_test.cc

@@ -452,6 +452,7 @@ TEST_F(TsFileTableReaderTest, TestDecoder) {
     ResultSet* ret = nullptr;
     int ret_value =
         reader.query("test_table", columns, INT64_MIN, INT64_MAX, ret);
+    ASSERT_EQ(ret_value, E_OK);
     auto* table_result_set = (storage::TableResultSet*)ret;
     bool has_next = false;
     int cur_lin = 0;

python/setup.py

@@ -25,7 +25,7 @@
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 
-version = "2.1.0.dev0"
+version = "2.2.0.dev"
 system = platform.system()