[manager] reconcile break-glass guard release

Author: shaohuaxi

kv_cache_manager/manager/migration_manager.cc

@@ -568,18 +568,37 @@ ErrorCode MigrationManager::BreakGlassReleaseAsyncCopy(const std::string &operat
     std::vector<std::vector<ErrorCode>> results;
     const auto ec = meta_searcher.BatchCASLocationStatus(
         request_context.get(), {record.block_key}, {{std::move(task)}}, results, true);
-    if (ec != EC_OK) {
-        return ec;
-    }
-    if (results.size() != 1 || results[0].size() != 1) {
-        return EC_MISMATCH;
-    }
-    if (results[0][0] != EC_OK) {
+    const auto transition_is_persistent = [&]() {
+        CacheLocationMapVector location_maps;
+        const auto get_result =
+            indexer->GetLocationsFromPersistent(request_context.get(), {record.block_key}, location_maps);
+        if (get_result.error_codes.size() != 1 || get_result.error_codes[0] != EC_OK || location_maps.size() != 1) {
+            return false;
+        }
+        const auto location_iter = location_maps[0].find(record.target_location_id);
+        return location_iter != location_maps[0].end() && location_iter->second &&
+               location_iter->second->status() == CLS_DELETING &&
+               !location_iter->second->has_migration_copy_guard();
+    };
+    const bool cas_applied = ec == EC_OK && results.size() == 1 && results[0].size() == 1 && results[0][0] == EC_OK;
+    if (cas_applied) {
+        // Sync timeout is an ambiguous durability result, not proof that the
+        // transition failed. Re-read the persistent backend before returning;
+        // otherwise a retry would CAS guarded WRITING after the first attempt
+        // had already durably produced guard-free DELETING and cleanup would
+        // be stranded forever.
+        if (!indexer->Sync({record.block_key}) && !transition_is_persistent()) {
+            return EC_ERROR;
+        }
+    } else if (!transition_is_persistent()) {
+        if (ec != EC_OK) {
+            return ec;
+        }
+        if (results.size() != 1 || results[0].size() != 1) {
+            return EC_MISMATCH;
+        }
         return results[0][0];
     }
-    if (!indexer->Sync({record.block_key})) {
-        return EC_ERROR;
-    }
 
     CopyTaskContext ctx;
     ctx.instance_group_name = record.instance_group_name;

kv_cache_manager/manager/test/migration_manager_test.cc

@@ -3737,7 +3737,11 @@ TEST_F(MigrationManagerTest, TestBreakGlassReleaseRequiresUnknownGuardAndClearsQ
     ASSERT_EQ(1u, mgr.ListAsyncCopyQuarantine(group_name).size());
     EXPECT_EQ(EC_BADARGS, mgr.BreakGlassReleaseAsyncCopy(operation_id, "", "ticket-89"));
     EXPECT_EQ(EC_NOENT, mgr.BreakGlassReleaseAsyncCopy("missing-operation", "operator-89", "ticket-89"));
+    g_guard_sync_call_count.store(0, std::memory_order_release);
+    Stub sync_stub;
+    sync_stub.set(ADDR(MetaIndexer, Sync), FailFirstGuardSyncStub);
     EXPECT_EQ(EC_OK, mgr.BreakGlassReleaseAsyncCopy(operation_id, "operator-89", "ticket-89"));
+    EXPECT_EQ(1, g_guard_sync_call_count.load(std::memory_order_acquire));
     EXPECT_TRUE(mgr.ListAsyncCopyQuarantine(group_name).empty());
     const auto stats = mgr.GetStats();
     EXPECT_EQ(0u, stats.async_copy_quarantine_operations);