Skip to content

Commit 65d4e59

Browse files
IvoDDIvoDD
committed
Fix tests
- Use `OutputFormat::NATIVE` for tests which register it - Small fix for date_range arrow to avoid reaching out of memory. Full fix will be in follow up PR - Skip unicode test for windows. To be fixed in follow up PR
1 parent 621c80d commit 65d4e59

File tree

5 files changed

+40
-38
lines changed

5 files changed

+40
-38
lines changed

‎cpp/arcticdb/column_store/test/ingestion_stress_test.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ TEST_F(IngestionStressStore, ScalarIntAppend) {
127127
auto read_query = std::make_shared<ReadQuery>();
128128
read_query->row_filter = universal_range();
129129
register_native_handler_data_factory();
130-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
130+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
131131
auto read_result = test_store_->read_dataframe_version(symbol, VersionQuery{}, read_query, ro, handler_data);
132132
GTEST_COUT << "columns in res: " << std::get<PandasOutputFrame>(read_result.frame_data).index_columns().size();
133133
}
@@ -215,7 +215,7 @@ TEST_F(IngestionStressStore, ScalarIntDynamicSchema) {
215215
auto read_query = std::make_shared<ReadQuery>();
216216
read_query->row_filter = universal_range();
217217
register_native_handler_data_factory();
218-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
218+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
219219
auto read_result = test_store_->read_dataframe_version_internal(symbol, VersionQuery{}, read_query, read_options, handler_data);
220220
}
221221

@@ -268,7 +268,7 @@ TEST_F(IngestionStressStore, DynamicSchemaWithStrings) {
268268
auto read_query = std::make_shared<ReadQuery>();
269269
read_query->row_filter = universal_range();
270270
register_native_handler_data_factory();
271-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
271+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
272272
auto read_result = test_store_->read_dataframe_version(symbol, VersionQuery{}, read_query, read_options, handler_data);
273273
ARCTICDB_DEBUG(log::version(), "result columns: {}", std::get<PandasOutputFrame>(read_result.frame_data).names());
274274
}

‎cpp/arcticdb/pipeline/read_frame.cpp

Lines changed: 24 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -371,32 +371,31 @@ ColumnTruncation get_truncate_range(
371371
const uint8_t* index_field_offset) {
372372
ColumnTruncation truncate_rows;
373373
if(read_options.output_format() == OutputFormat::ARROW) {
374-
util::variant_match(read_query.row_filter,
375-
[&truncate_rows, &frame, &context, &index_field, index_field_offset, encoding_version] (const IndexRange& index_range) {
376-
const auto& time_range = static_cast<const TimestampRange&>(index_range);
377-
const auto& slice_time_range = context.slice_and_key().key().time_range();
378-
if(contains(slice_time_range, time_range.first) || contains(slice_time_range, time_range.second)) {
379-
if(context.fetch_index()) {
380-
const auto& index_column = frame.column(0);
381-
const auto& current_row_range = context.slice_and_key().slice().row_range;
382-
truncate_rows = get_truncate_range_from_index(index_column, time_range.first, time_range.second, current_row_range.first, current_row_range.second);
383-
} else {
384-
const auto& frame_index_desc = frame.descriptor().fields(0UL);
385-
Column sink{frame_index_desc.type(), encoding_sizes::field_uncompressed_size(index_field), AllocationType::PRESIZED, Sparsity::PERMITTED};
386-
std::optional<util::BitMagic> bv;
387-
(void)decode_field(frame_index_desc.type(), index_field, index_field_offset, sink, bv, encoding_version);
388-
truncate_rows = get_truncate_range_from_index(sink, time_range.first, time_range.second);
389-
}
390-
}
374+
util::variant_match(read_query.row_filter,
375+
[&truncate_rows, &frame, &context, &index_field, index_field_offset, encoding_version] (const IndexRange& index_range) {
376+
const auto& time_range = static_cast<const TimestampRange&>(index_range);
377+
const auto& slice_time_range = context.slice_and_key().key().time_range();
378+
if(contains(slice_time_range, time_range.first) || contains(slice_time_range, time_range.second)) {
379+
if(context.fetch_index()) {
380+
const auto& index_column = frame.column(0);
381+
truncate_rows = get_truncate_range_from_index(index_column, time_range.first, time_range.second);
382+
} else {
383+
const auto& frame_index_desc = frame.descriptor().fields(0UL);
384+
Column sink{frame_index_desc.type(), encoding_sizes::field_uncompressed_size(index_field), AllocationType::PRESIZED, Sparsity::PERMITTED};
385+
std::optional<util::BitMagic> bv;
386+
(void)decode_field(frame_index_desc.type(), index_field, index_field_offset, sink, bv, encoding_version);
387+
truncate_rows = get_truncate_range_from_index(sink, time_range.first, time_range.second);
388+
}
389+
}
391390
},
392-
[&context] (const RowRange& row_range) {
393-
const auto& slice_row_range = context.slice_and_key().slice().row_range;
394-
get_truncate_range_from_rows(row_range, slice_row_range.start(), slice_row_range.end());
395-
},
396-
[] (const auto&) {
397-
// Do nothing
398-
});
399-
}
391+
[&context] (const RowRange& row_range) {
392+
const auto& slice_row_range = context.slice_and_key().slice().row_range;
393+
get_truncate_range_from_rows(row_range, slice_row_range.start(), slice_row_range.end());
394+
},
395+
[] (const auto&) {
396+
// Do nothing
397+
});
398+
}
400399
return truncate_rows;
401400
};
402401

‎cpp/arcticdb/storage/test/test_memory_storage.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ TEST(InMemory, ReadTwice) {
2929

3030
auto read_query = std::make_shared<ReadQuery>();
3131
register_native_handler_data_factory();
32-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
32+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
3333
auto read_result1 = version_store.read_dataframe_version_internal(symbol, VersionQuery{}, read_query, ReadOptions{}, handler_data);
3434
auto read_result2 = version_store.read_dataframe_version_internal(symbol, VersionQuery{}, read_query, ReadOptions{}, handler_data);
3535
}

‎cpp/arcticdb/version/test/test_sparse.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ TEST_F(SparseTestStore, SimpleRoundtrip) {
8888
auto read_query = std::make_shared<ReadQuery>();
8989
read_query->row_filter = universal_range();
9090
register_native_handler_data_factory();
91-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
91+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
9292
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
9393
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
9494

@@ -180,7 +180,7 @@ TEST_F(SparseTestStore, SimpleRoundtripBackwardsCompat) {
180180
auto read_query = std::make_shared<ReadQuery>();
181181
read_query->row_filter = universal_range();
182182
register_native_handler_data_factory();
183-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
183+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
184184
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
185185
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
186186

@@ -231,7 +231,7 @@ TEST_F(SparseTestStore, DenseToSparse) {
231231
auto read_query = std::make_shared<ReadQuery>();
232232
read_query->row_filter = universal_range();
233233
register_native_handler_data_factory();
234-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
234+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
235235
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
236236
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
237237

@@ -334,7 +334,7 @@ TEST_F(SparseTestStore, Multiblock) {
334334
auto read_query = std::make_shared<ReadQuery>();
335335
read_query->row_filter = universal_range();
336336
register_native_handler_data_factory();
337-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
337+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
338338
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
339339
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
340340

@@ -387,7 +387,7 @@ TEST_F(SparseTestStore, Segment) {
387387
auto read_query = std::make_shared<ReadQuery>();
388388
read_query->row_filter = universal_range();
389389
register_native_handler_data_factory();
390-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
390+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
391391
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
392392
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
393393

@@ -447,7 +447,7 @@ TEST_F(SparseTestStore, SegmentWithExistingIndex) {
447447
auto read_query = std::make_shared<ReadQuery>();
448448
read_query->row_filter = universal_range();
449449
register_native_handler_data_factory();
450-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
450+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
451451
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
452452
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
453453

@@ -508,7 +508,7 @@ TEST_F(SparseTestStore, SegmentAndFilterColumn) {
508508
read_query->columns = {"time", "first"};
509509
read_query->row_filter = universal_range();
510510
register_native_handler_data_factory();
511-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
511+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
512512
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
513513
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
514514
ASSERT_EQ(frame.row_count(), num_rows);
@@ -564,7 +564,7 @@ TEST_F(SparseTestStore, SegmentWithRangeFilter) {
564564
auto read_query = std::make_shared<ReadQuery>();
565565
read_query->row_filter = IndexRange(timestamp{3000}, timestamp{6999});
566566
register_native_handler_data_factory();
567-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
567+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
568568
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
569569
const auto& frame =std::get<PandasOutputFrame>(read_result.frame_data).frame();;
570570

@@ -618,7 +618,7 @@ TEST_F(SparseTestStore, Compact) {
618618
auto read_query = std::make_shared<ReadQuery>();
619619
read_query->row_filter = universal_range();
620620
register_native_handler_data_factory();
621-
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::PANDAS);
621+
auto handler_data = TypeHandlerRegistry::instance()->get_handler_data(OutputFormat::NATIVE);
622622
auto read_result = test_store_->read_dataframe_version(stream_id, pipelines::VersionQuery{}, read_query, read_options, handler_data);
623623
const auto& frame = std::get<PandasOutputFrame>(read_result.frame_data).frame();
624624

‎python/tests/unit/arcticdb/version_store/test_arrow.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import pyarrow as pa
99
from arcticdb.util.test import get_sample_dataframe
1010
from arcticdb_ext.storage import KeyType
11+
from tests.util.mark import WINDOWS
1112

1213

1314
def test_basic(lmdb_version_store_v1):
@@ -117,6 +118,8 @@ def test_strings_multiple_segments_and_columns(lmdb_version_store_tiny_segment,
117118
assert_frame_equal(result, df)
118119

119120

121+
# TODO: Fix unicode strings on windows
122+
@pytest.mark.skipif(WINDOWS, reason="Unicode arrow strings fail on windows")
120123
def test_all_types(lmdb_version_store_v1):
121124
lib = lmdb_version_store_v1
122125
# sample dataframe contains all dtypes + unicode strings

0 commit comments

Comments
 (0)