Skip to content

Commit 9dddedf

Browse files
committed
[fix](cloud) Exclude stale rowsets from schema change temporary tablet
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: During merge-on-write cloud schema changes, the delete-bitmap temporary tablet copied stale rowset metadata from the real shadow tablet. Its version tracker could select a compacted stale edge that was absent from the active rowset map, causing schema change to fail with a rowset capture error under concurrent writes and compaction. Clear only the copied stale rowset metadata before constructing the temporary tablet so its tracker is built exclusively from active schema-change output and incremental rowsets, without altering the source tablet or shared delete-bitmap cache. ### Release note Fix cloud merge-on-write schema changes that could fail during concurrent writes and compaction. ### Check List (For Author) - Test: Unit Test - `./run-be-ut.sh --run --filter=CloudSchemaChangeJobTest.* -j 48` - `./build.sh --be -j 48` - Behavior changed: Yes. Cloud schema change temporary tablets no longer inherit stale rowset metadata. - Does this need documentation: No
1 parent ad7e343 commit 9dddedf

4 files changed

Lines changed: 73 additions & 2 deletions

File tree

be/src/cloud/cloud_schema_change_job.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,9 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
579579
.tag("alter_version", alter_version);
580580
RETURN_IF_ERROR(_cloud_storage_engine.register_compaction_stop_token(_new_tablet, initiator));
581581
TabletMetaSharedPtr tmp_meta = std::make_shared<TabletMeta>(*(_new_tablet->tablet_meta()));
582+
// The temporary tablet must build its version graph only from active rowsets. Stale
583+
// rowsets copied from the real tablet are not present in its active rowset map.
584+
tmp_meta->clear_stale_rs_metas();
582585
tmp_meta->delete_bitmap().delete_bitmap.clear();
583586
// Keep only version [0-1] rowset, other rowsets will be added in _output_rowsets
584587
auto& rs_metas = tmp_meta->all_mutable_rs_metas();

be/src/storage/tablet/tablet_meta.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1171,12 +1171,16 @@ Status TabletMeta::set_partition_id(int64_t partition_id) {
11711171
}
11721172

11731173
void TabletMeta::clear_stale_rowset() {
1174-
_stale_rs_metas.clear();
1174+
clear_stale_rs_metas();
11751175
if (_enable_unique_key_merge_on_write) {
11761176
_delete_bitmap->clear_rowset_cache_version();
11771177
}
11781178
}
11791179

1180+
void TabletMeta::clear_stale_rs_metas() {
1181+
_stale_rs_metas.clear();
1182+
}
1183+
11801184
void TabletMeta::clear_rowsets() {
11811185
_rs_metas.clear();
11821186
if (_enable_unique_key_merge_on_write) {

be/src/storage/tablet/tablet_meta.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,9 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
223223
// used for after tablet cloned to clear stale rowset
224224
void clear_stale_rowset();
225225

226+
// Clear stale rowset metadata without changing the delete bitmap cache.
227+
void clear_stale_rs_metas();
228+
226229
void clear_rowsets();
227230

228231
// MUST hold EXCLUSIVE `_meta_lock` in belonged Tablet

be/test/cloud/cloud_schema_change_job_test.cpp

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,10 @@ class CloudSchemaChangeJobTest : public testing::Test {
153153

154154
protected:
155155
RowsetSharedPtr create_rowset(TabletSchemaSPtr schema, int64_t tablet_id, int64_t start,
156-
int64_t end) {
156+
int64_t end, int64_t rowset_id = 540081) const {
157157
RowsetMetaPB pb;
158158
json2pb::JsonToProtoMessage(_json_rowset_meta, &pb);
159+
pb.set_rowset_id(rowset_id);
159160
pb.set_tablet_id(tablet_id);
160161
pb.set_start_version(start);
161162
pb.set_end_version(end);
@@ -173,6 +174,66 @@ class CloudSchemaChangeJobTest : public testing::Test {
173174
std::shared_ptr<CloudClusterInfo> _cluster_info;
174175
};
175176

177+
// GTest assertion macros inflate cognitive complexity for this linear scenario.
178+
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
179+
TEST_F(CloudSchemaChangeJobTest, DeleteBitmapTmpTabletDoesNotInheritStaleRowsets) {
180+
constexpr int64_t new_tablet_id = 60002;
181+
182+
TabletMetaSharedPtr new_meta(new TabletMeta(
183+
1, 2, new_tablet_id, new_tablet_id + 100, 4, 5, TTabletSchema(), 6, {{7, 8}},
184+
UniqueId(11, 12), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, -1, true));
185+
auto placeholder = create_rowset(new_meta->tablet_schema(), new_tablet_id, 0, 1, 6000201);
186+
auto compacted = create_rowset(new_meta->tablet_schema(), new_tablet_id, 2, 3, 6000202);
187+
auto sc_output_2 = create_rowset(new_meta->tablet_schema(), new_tablet_id, 2, 2, 6000203);
188+
auto sc_output_3 = create_rowset(new_meta->tablet_schema(), new_tablet_id, 3, 3, 6000204);
189+
auto sc_output_4 = create_rowset(new_meta->tablet_schema(), new_tablet_id, 4, 4, 6000205);
190+
ASSERT_NE(placeholder, nullptr);
191+
ASSERT_NE(compacted, nullptr);
192+
ASSERT_NE(sc_output_2, nullptr);
193+
ASSERT_NE(sc_output_3, nullptr);
194+
ASSERT_NE(sc_output_4, nullptr);
195+
196+
ASSERT_TRUE(new_meta->add_rs_meta(placeholder->rowset_meta()).ok());
197+
ASSERT_TRUE(new_meta->add_rs_meta(compacted->rowset_meta()).ok());
198+
new_meta->modify_rs_metas({}, {compacted->rowset_meta()});
199+
ASSERT_EQ(new_meta->all_stale_rs_metas().size(), 1);
200+
201+
auto new_tablet = std::make_shared<CloudTablet>(_engine, new_meta);
202+
auto* sp = SyncPoint::get_instance();
203+
sp->clear_all_call_backs();
204+
sp->enable_processing();
205+
sp->set_call_back("CloudMetaMgr::prepare_tablet_job", [](auto&& outcome) {
206+
auto* pairs = try_any_cast_ret<Status>(outcome);
207+
pairs->second = true;
208+
pairs->first = Status::OK();
209+
auto* resp = try_any_cast<cloud::StartTabletJobResponse*>(outcome[1]);
210+
resp->mutable_status()->set_code(cloud::MetaServiceCode::OK);
211+
});
212+
213+
Status captured_status = Status::InternalError("temporary tablet was not inspected");
214+
RowsetIdUnorderedSet captured_rowset_ids;
215+
sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& outcome) {
216+
auto* tablet = try_any_cast<CloudTablet*>(outcome[0]);
217+
std::shared_lock rlock(tablet->get_header_lock());
218+
captured_status = tablet->get_all_rs_id_unlocked(4, &captured_rowset_ids);
219+
auto* pairs = try_any_cast_ret<Status>(outcome);
220+
pairs->second = true;
221+
pairs->first = Status::InternalError("stop after inspecting temporary tablet");
222+
});
223+
224+
CloudSchemaChangeJob sc_job(_engine, "test_tmp_tablet_stale_rowsets", 9999999999);
225+
sc_job._new_tablet = new_tablet;
226+
sc_job._output_rowsets = {sc_output_2, sc_output_3, sc_output_4};
227+
auto status = sc_job._process_delete_bitmap(4, 5, 12345, "");
228+
ASSERT_TRUE(_engine.unregister_compaction_stop_token(new_tablet, false).ok());
229+
230+
ASSERT_FALSE(status.ok());
231+
ASSERT_NE(status.to_string().find("stop after inspecting temporary tablet"), std::string::npos);
232+
ASSERT_TRUE(captured_status.ok()) << captured_status.to_string();
233+
ASSERT_EQ(captured_rowset_ids.size(), 3);
234+
ASSERT_EQ(new_meta->all_stale_rs_metas().size(), 1);
235+
}
236+
176237
TEST_F(CloudSchemaChangeJobTest, FillVersionHolesBeforeNewTabletRunning) {
177238
int64_t base_tablet_id = 40001;
178239
int64_t new_tablet_id = 40002;

0 commit comments

Comments
 (0)