File tree Expand file tree Collapse file tree 3 files changed +21
-9
lines changed Expand file tree Collapse file tree 3 files changed +21
-9
lines changed Original file line number Diff line number Diff line change @@ -187,7 +187,7 @@ FieldStatsImpl generate_column_statistics(ColumnData column_data) {
187
187
const size_t count = block->row_count ();
188
188
if constexpr (is_numeric_type (TagType::DataTypeTag::data_type)) {
189
189
return generate_numeric_statistics<RawType>(std::span{ptr, count});
190
- } else if constexpr (is_dynamic_string_type (TagType::DataTypeTag::data_type) && !is_output_only_type (TagType::DataTypeTag::data_type)) {
190
+ } else if constexpr (is_dynamic_string_type (TagType::DataTypeTag::data_type) && !is_arrow_output_only_type (TagType::DataTypeTag::data_type)) {
191
191
return generate_string_statistics (std::span{ptr, count});
192
192
} else {
193
193
util::raise_rte (" Cannot generate statistics for data type" );
@@ -200,7 +200,7 @@ FieldStatsImpl generate_column_statistics(ColumnData column_data) {
200
200
if constexpr (is_numeric_type (TagType::DataTypeTag::data_type)) {
201
201
auto local_stats = generate_numeric_statistics<RawType>(std::span{ptr, count});
202
202
stats.compose <RawType>(local_stats);
203
- } else if constexpr (is_dynamic_string_type (TagType::DataTypeTag::data_type) && !is_output_only_type (TagType::DataTypeTag::data_type)) {
203
+ } else if constexpr (is_dynamic_string_type (TagType::DataTypeTag::data_type) && !is_arrow_output_only_type (TagType::DataTypeTag::data_type)) {
204
204
auto local_stats = generate_string_statistics (std::span{ptr, count});
205
205
stats.compose <RawType>(local_stats);
206
206
} else {
Original file line number Diff line number Diff line change @@ -297,7 +297,7 @@ constexpr bool is_dynamic_string_type(DataType v) {
297
297
return is_dynamic_string_type (slice_value_type (v));
298
298
}
299
299
300
- constexpr bool is_output_only_type (DataType d) {
300
+ constexpr bool is_arrow_output_only_type (DataType d) {
301
301
return d == DataType::UTF_DYNAMIC32;
302
302
}
303
303
@@ -500,6 +500,10 @@ constexpr bool is_object_type(TypeDescriptor td) {
500
500
|| is_array_type (td);
501
501
}
502
502
503
+ constexpr bool is_arrow_output_only_type (TypeDescriptor td) {
504
+ return is_arrow_output_only_type (td.data_type ());
505
+ }
506
+
503
507
inline void set_data_type (DataType data_type, TypeDescriptor &type_desc) {
504
508
type_desc.data_type_ = data_type;
505
509
}
Original file line number Diff line number Diff line change @@ -40,21 +40,29 @@ PandasOutputFrame::~PandasOutputFrame() {
40
40
column_data.type ().visit_tag ([&](auto type_desc_tag) {
41
41
using TDT = decltype (type_desc_tag);
42
42
constexpr auto td = TypeDescriptor (type_desc_tag);
43
- if constexpr (is_object_type (TypeDescriptor (type_desc_tag) )) {
43
+ if constexpr (is_object_type (td )) {
44
44
if constexpr (is_array_type (td)) {
45
45
auto it = column_data.buffer ().iterator (sizeof (PyObject*));
46
46
while (!it.finished ()) {
47
- util::check (reinterpret_cast <PyObject*>(it.value ()) != nullptr , " Can't delete null item" );
48
- Py_DECREF (reinterpret_cast <PyObject*>(it.value ()));
47
+ if (reinterpret_cast <PyObject*>(it.value ()) != nullptr ) {
48
+ Py_DECREF (reinterpret_cast <PyObject*>(it.value ()));
49
+ } else {
50
+ log::version ().error (" Unexpected nullptr to DecRef in PandasOutputFrame destructor" );
51
+ }
49
52
it.next ();
50
53
}
51
- } else {
54
+ } else if constexpr (! is_arrow_output_only_type (td)) {
52
55
while (auto block = column_data.next <TDT>()) {
53
56
for (auto item : *block) {
54
- util::check (reinterpret_cast <PyObject *>(item) != nullptr , " Can't delete null item" );
55
- Py_DECREF (reinterpret_cast <PyObject *>(item));
57
+ if (reinterpret_cast <PyObject*>(item) != nullptr ) {
58
+ Py_DECREF (reinterpret_cast <PyObject*>(item));
59
+ } else {
60
+ log::version ().error (" Unexpected nullptr to DecRef in PandasOutputFrame destructor" );
61
+ }
56
62
}
57
63
}
64
+ } else {
65
+ log::version ().error (" Unexpected arrow output format seen in PandasOutputFrame" );
58
66
}
59
67
}
60
68
});
You can’t perform that action at this time.
0 commit comments