Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](meta) Add strict field synchronization checks for cloud/non-cloud mode metadata conversions to prevent inconsistent PB bugs #48837

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[opt](meta) Add strict field synchronization checks for cloud/non-cloud mode metadata conversions to prevent inconsistent PB bugs #48837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions be/src/cloud/pb_convert.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
out->set_schema_version(in.schema_version());
}
out->set_enable_segments_file_size(in.enable_segments_file_size());
out->set_has_variant_type_in_schema(in.has_has_variant_type_in_schema());
out->set_has_variant_type_in_schema(in.has_variant_type_in_schema());
out->set_enable_inverted_index_file_info(in.enable_inverted_index_file_info());
out->set_compaction_level(in.compaction_level());
out->mutable_inverted_index_file_info()->CopyFrom(in.inverted_index_file_info());
Expand Down Expand Up @@ -239,6 +239,7 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in,
out->set_schema_version(in.schema_version());
}
out->set_enable_segments_file_size(in.enable_segments_file_size());
out->set_has_variant_type_in_schema(in.has_variant_type_in_schema());
out->set_enable_inverted_index_file_info(in.enable_inverted_index_file_info());
out->set_compaction_level(in.compaction_level());
out->mutable_inverted_index_file_info()->CopyFrom(in.inverted_index_file_info());
Expand Down Expand Up @@ -295,6 +296,7 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in,
out->set_schema_version(in.schema_version());
}
out->set_enable_segments_file_size(in.enable_segments_file_size());
out->set_has_variant_type_in_schema(in.has_variant_type_in_schema());
out->set_enable_inverted_index_file_info(in.enable_inverted_index_file_info());
out->set_compaction_level(in.compaction_level());
out->mutable_inverted_index_file_info()->Swap(in.mutable_inverted_index_file_info());
Expand Down Expand Up @@ -340,9 +342,11 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
out->set_is_dynamic_schema(in.is_dynamic_schema());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}

void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in) {
Expand All @@ -369,9 +373,11 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
out->set_is_dynamic_schema(in.is_dynamic_schema());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_enable_variant_flatten_nested(in.variant_enable_flatten_nested());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}

TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB& in) {
Expand Down Expand Up @@ -411,9 +417,11 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
out->set_is_dynamic_schema(in.is_dynamic_schema());
out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}

void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in) {
Expand Down Expand Up @@ -441,9 +449,11 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
out->set_is_dynamic_schema(in.is_dynamic_schema());
out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
out->set_variant_enable_flatten_nested(in.enable_variant_flatten_nested());
out->set_enable_variant_flatten_nested(in.enable_variant_flatten_nested());
out->set_skip_bitmap_col_idx(in.skip_bitmap_col_idx());
out->set_storage_page_size(in.storage_page_size());
out->set_is_in_memory(in.is_in_memory());
out->set_row_store_page_size(in.row_store_page_size());
}

TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB& in) {
Expand Down Expand Up @@ -477,7 +487,13 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in)
doris_rowset_meta_to_cloud(out->add_rs_metas(), rs_meta);
}
}
// ATTN: inc_rs_metas are deprecated, ignored here.
// ATTN: inc_rs_metas are deprecated, here is for conversion check
if (in.inc_rs_metas_size()) {
out->mutable_inc_rs_metas()->Reserve(in.inc_rs_metas_size());
for (const auto& rs_meta : in.inc_rs_metas()) {
doris_rowset_meta_to_cloud(out->add_inc_rs_metas(), rs_meta);
}
}
if (in.has_alter_task()) {
out->mutable_alter_task()->CopyFrom(in.alter_task());
}
Expand Down Expand Up @@ -541,7 +557,15 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
doris_rowset_meta_to_cloud(out->add_rs_metas(), std::move(*in.mutable_rs_metas(i)));
}
}
// ATTN: inc_rs_metas are deprecated, ignored here.
// ATTN: inc_rs_metas are deprecated, here is for conversion check
if (in.inc_rs_metas_size()) {
size_t rs_metas_size = in.inc_rs_metas_size();
out->mutable_inc_rs_metas()->Reserve(rs_metas_size);
for (size_t i = 0; i < rs_metas_size; ++i) {
doris_rowset_meta_to_cloud(out->add_inc_rs_metas(),
std::move(*in.mutable_inc_rs_metas(i)));
}
}
if (in.has_alter_task()) {
out->mutable_alter_task()->Swap(in.mutable_alter_task());
}
Expand Down Expand Up @@ -618,7 +642,13 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in)
cloud_rowset_meta_to_doris(out->add_rs_metas(), rs_meta);
}
}
// ATTN: inc_rs_metas are deprecated, ignored here.
// ATTN: inc_rs_metas are deprecated, here is for conversion check
if (in.inc_rs_metas_size()) {
out->mutable_inc_rs_metas()->Reserve(in.inc_rs_metas_size());
for (const auto& rs_meta : in.inc_rs_metas()) {
cloud_rowset_meta_to_doris(out->add_inc_rs_metas(), rs_meta);
}
}
if (in.has_alter_task()) {
out->mutable_alter_task()->CopyFrom(in.alter_task());
}
Expand Down Expand Up @@ -682,7 +712,15 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
cloud_rowset_meta_to_doris(out->add_rs_metas(), std::move(*in.mutable_rs_metas(i)));
}
}
// ATTN: inc_rs_metas are deprecated, ignored here.
// ATTN: inc_rs_metas are deprecated, here is for conversion check
if (in.inc_rs_metas_size()) {
size_t rs_metas_size = in.inc_rs_metas_size();
out->mutable_inc_rs_metas()->Reserve(rs_metas_size);
for (size_t i = 0; i < rs_metas_size; i++) {
cloud_rowset_meta_to_doris(out->add_inc_rs_metas(),
std::move(*in.mutable_inc_rs_metas(i)));
}
}
if (in.has_alter_task()) {
out->mutable_alter_task()->Swap(in.mutable_alter_task());
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/tablet_meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
}

if (tablet_schema.__isset.variant_enable_flatten_nested) {
schema->set_variant_enable_flatten_nested(tablet_schema.variant_enable_flatten_nested);
schema->set_enable_variant_flatten_nested(tablet_schema.variant_enable_flatten_nested);
}

if (tablet_schema.__isset.enable_single_replica_compaction) {
Expand Down
8 changes: 4 additions & 4 deletions be/src/olap/tablet_schema.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac

_row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(),
schema.row_store_column_unique_ids().end());
_variant_enable_flatten_nested = schema.variant_enable_flatten_nested();
_enable_variant_flatten_nested = schema.enable_variant_flatten_nested();
_vl_field_mem_size += _row_store_column_unique_ids.capacity() * sizeof(int32_t);
update_metadata_size();
}
Expand Down Expand Up @@ -1139,7 +1139,7 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version
_sort_col_num = ori_tablet_schema.sort_col_num();
_row_store_page_size = ori_tablet_schema.row_store_page_size();
_storage_page_size = ori_tablet_schema.storage_page_size();
_variant_enable_flatten_nested = ori_tablet_schema.variant_flatten_nested();
_enable_variant_flatten_nested = ori_tablet_schema.variant_flatten_nested();

// copy from table_schema_param
_schema_version = version;
Expand Down Expand Up @@ -1304,7 +1304,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign(
_row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end());
tablet_schema_pb->set_variant_enable_flatten_nested(_variant_enable_flatten_nested);
tablet_schema_pb->set_enable_variant_flatten_nested(_enable_variant_flatten_nested);
}

size_t TabletSchema::row_size() const {
Expand Down Expand Up @@ -1573,7 +1573,7 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
if (a._row_store_page_size != b._row_store_page_size) return false;
if (a._storage_page_size != b._storage_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false;
if (a._variant_enable_flatten_nested != b._variant_enable_flatten_nested) return false;
if (a._enable_variant_flatten_nested != b._enable_variant_flatten_nested) return false;
return true;
}

Expand Down
8 changes: 4 additions & 4 deletions be/src/olap/tablet_schema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,10 +371,10 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
_disable_auto_compaction = disable_auto_compaction;
}
bool disable_auto_compaction() const { return _disable_auto_compaction; }
void set_variant_enable_flatten_nested(bool flatten_nested) {
_variant_enable_flatten_nested = flatten_nested;
void set_enable_variant_flatten_nested(bool flatten_nested) {
_enable_variant_flatten_nested = flatten_nested;
}
bool variant_flatten_nested() const { return _variant_enable_flatten_nested; }
bool variant_flatten_nested() const { return _enable_variant_flatten_nested; }
void set_enable_single_replica_compaction(bool enable_single_replica_compaction) {
_enable_single_replica_compaction = enable_single_replica_compaction;
}
Expand Down Expand Up @@ -594,7 +594,7 @@ class TabletSchema : public MetadataAdder<TabletSchema> {
// Contains column ids of which columns should be encoded into row store.
// ATTN: For compability reason empty cids means all columns of tablet schema are encoded to row column
std::vector<int32_t> _row_store_column_unique_ids;
bool _variant_enable_flatten_nested = false;
bool _enable_variant_flatten_nested = false;
int64_t _vl_field_mem_size {0}; // variable length field
};

Expand Down
Loading
Loading