From c7fb059ece8195a80c74c9e45f5f9df24b1f34c7 Mon Sep 17 00:00:00 2001 From: Satya Bodapati Date: Wed, 30 Jul 2025 15:26:07 +0100 Subject: [PATCH 1/2] PXB-3571 : --transition-key does not save keys with --lock-ddl=reduced Problem: -------- If --transition-key is used with reduced lock backups (--lock-ddl=reduced), it doesn't save the tablespace encryption key of all tablespaces. This is because, as part of reduced lock, we close all tablespaces and re-open only the tablespaces that are involved in DDLs. See handle_ddl_operations(). Later, we iterate over the in-memory list of tablespaces and save the keys. Since the DDL tracker closed and removed all the tablespaces, it will now save only the keys of tablespaces opened by DDL tracker. Fix: ---- We now divide the tablespace keys into multiple stages. The old xb_tablespace_keys_dump() is now refactored to TablespaceKeyDumper and functionality is split as below: 1. Saving keys from in-memory list: This is split into two phases. We do once before tablespaces are removed from cache. And again, at the end for the new tablespace discovered by reduced lock. This is dump_from_spaces() 2. Saving keys from redo: This can be done only after redo thread is stopped. So it is done at end. This is dump_from_redo(). 3. Saving keys from encryption_infos. This encryption_infos contains the keys discovered for encrypted redo log. This is done as usual at end. The function that does this is dump_from_encryption_infos() --- .../innobase/xtrabackup/src/backup_copy.cc | 11 + .../innobase/xtrabackup/src/backup_mysql.h | 3 + .../innobase/xtrabackup/src/ddl_tracker.cc | 5 + storage/innobase/xtrabackup/src/ddl_tracker.h | 4 + .../xtrabackup/src/keyring_plugins.cc | 289 ++++++++++-------- .../innobase/xtrabackup/src/keyring_plugins.h | 61 +++- storage/innobase/xtrabackup/src/xtrabackup.cc | 37 ++- .../suites/reducedlock/transition_keys.sh | 30 ++ .../reducedlock/transition_keys_debug.sh | 67 ++++ 9 files changed, 365 insertions(+), 142 deletions(-) create mode 100644 storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys.sh create mode 100644 storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys_debug.sh diff --git a/storage/innobase/xtrabackup/src/backup_copy.cc b/storage/innobase/xtrabackup/src/backup_copy.cc index 77d816ec1ffe..0c95b0731a95 100644 --- a/storage/innobase/xtrabackup/src/backup_copy.cc +++ b/storage/innobase/xtrabackup/src/backup_copy.cc @@ -1406,6 +1406,17 @@ bool backup_start(Backup_context &context) { /* LTFB/LIFB has to be executed before copying MyISAM */ if (ddl_tracker != nullptr) { debug_sync_point("ddl_tracker_before_lock_ddl"); + + /* The tablespaces will be closed on handle_ddl_operations. Hence + dump the tablespace keys now. For tablespace that are tracked as + dropped, dont dump the tablespace encryption keys. Note that we + still need to save tablespace keys found from redo. We cannot do it + now as the redo log thread is still in progress. It is done after + the redo thread is stopped. See TablespaceKeyDumper::dump_from_redo() */ + if (context.ts_key_dumper != nullptr) { + context.ts_key_dumper->dump_from_spaces(true); + } + if (!lock_tables_for_backup(mysql_connection, opt_backup_lock_timeout, opt_backup_lock_retry_count)) { return (false); diff --git a/storage/innobase/xtrabackup/src/backup_mysql.h b/storage/innobase/xtrabackup/src/backup_mysql.h index be2e1dac9df9..851622dcca2d 100644 --- a/storage/innobase/xtrabackup/src/backup_mysql.h +++ b/storage/innobase/xtrabackup/src/backup_mysql.h @@ -145,10 +145,13 @@ class Myrocks_checkpoint { file_list data_files() const; }; +class TablespaceKeyDumper; + struct Backup_context { Myrocks_checkpoint myrocks_checkpoint; std::unordered_set rocksdb_files; Redo_Log_Data_Manager *redo_mgr; + TablespaceKeyDumper *ts_key_dumper; }; /* server capabilities */ diff --git a/storage/innobase/xtrabackup/src/ddl_tracker.cc b/storage/innobase/xtrabackup/src/ddl_tracker.cc index d21485a754f2..76843c43c449 100644 --- a/storage/innobase/xtrabackup/src/ddl_tracker.cc +++ b/storage/innobase/xtrabackup/src/ddl_tracker.cc @@ -255,6 +255,11 @@ void ddl_tracker_t::add_drop_table_from_redo(const space_id_t space_id, << " delete space ID: " << space_id << " Name: " << new_space_name; } +bool ddl_tracker_t::is_tablespace_dropped(const space_id_t space_id) { + std::lock_guard lock(m_ddl_tracker_mutex); + return (drops.find(space_id) != drops.end()); +} + void ddl_tracker_t::add_rename_ibd_scan(const space_id_t &space_id, std::string new_name) { // undo tablespaces are tracked separately. diff --git a/storage/innobase/xtrabackup/src/ddl_tracker.h b/storage/innobase/xtrabackup/src/ddl_tracker.h index 58c294c2e941..ae0d07ef0a2d 100644 --- a/storage/innobase/xtrabackup/src/ddl_tracker.h +++ b/storage/innobase/xtrabackup/src/ddl_tracker.h @@ -155,6 +155,10 @@ class ddl_tracker_t { @param[in] space_id tablespace identifier @param[in] new_name tablespace new name */ void add_rename_ibd_scan(const space_id_t &space_id, std::string new_name); + + /** @return true if tablespace is dropped + @param[in] space_id tablespace id */ + bool is_tablespace_dropped(const space_id_t space_id); }; /** Insert into meta files map. This map is later used to delete the right diff --git a/storage/innobase/xtrabackup/src/keyring_plugins.cc b/storage/innobase/xtrabackup/src/keyring_plugins.cc index 6d169078565a..a81404a29720 100644 --- a/storage/innobase/xtrabackup/src/keyring_plugins.cc +++ b/storage/innobase/xtrabackup/src/keyring_plugins.cc @@ -38,6 +38,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #include "keyring_operations_helper.h" #include "backup_mysql.h" +#include "ddl_tracker.h" #include "keyring_plugins.h" #include "rpl_log_encryption.h" #include "utils.h" @@ -186,17 +187,6 @@ dberr_t xb_set_encryption(fil_space_t *space) { return (fil_set_encryption(space->id, Encryption::AES, key, iv)); } -#define TRANSITION_KEY_PREFIX_STR "XBKey" - -const char *TRANSITION_KEY_PREFIX = TRANSITION_KEY_PREFIX_STR; -const size_t TRANSITION_KEY_PREFIX_LEN = sizeof(TRANSITION_KEY_PREFIX_STR) - 1; -const size_t TRANSITION_KEY_RANDOM_DATA_LEN = 32; -const size_t TRANSITION_KEY_NAME_MAX_LEN_V1 = - Encryption::SERVER_UUID_LEN + 2 + 45; -const size_t TRANSITION_KEY_NAME_MAX_LEN_V2 = - TRANSITION_KEY_PREFIX_LEN + Encryption::SERVER_UUID_LEN + - TRANSITION_KEY_RANDOM_DATA_LEN + 1; - /** Fetch the key from keyring. @param[in] key_name key name @param[out] key key @@ -250,9 +240,14 @@ static bool xb_create_transition_key(char *key_name, char *key) { base64_encode(rand32, 20, rand64); + std::ostringstream oss; + /* Trasnsition key name is composed of server uuid and random suffix. */ - snprintf(key_name, TRANSITION_KEY_NAME_MAX_LEN_V2, "%s-%s-%s", - TRANSITION_KEY_PREFIX, server_uuid, rand64); + oss << TRANSITION_KEY_PREFIX << '-' << server_uuid << '-' << rand64; + std::string s = oss.str(); + memset(key_name, 0, TRANSITION_KEY_NAME_MAX_LEN_V2); + std::memcpy(key_name, s.data(), + std::min(s.size(), TRANSITION_KEY_NAME_MAX_LEN_V2)); /* Let keyring generate key for us. */ ret = srv_keyring_generator->generate(key_name, nullptr, "AES", @@ -661,120 +656,6 @@ static bool xb_tablespace_keys_write_single(ds_file_t *stream, return (true); } -/** Dump tablespace keys into encrypted "xtrabackup_keys" file. -@param[in] ds_ctxt datasink context to output file into -@param[in] transition_key transition key used to encrypt - tablespace keys -@param[in] transition_key_len transition key length -@return true if success */ -bool xb_tablespace_keys_dump(ds_ctxt_t *ds_ctxt, const char *transition_key, - size_t transition_key_len) { - byte derived_key[Encryption::KEY_LEN]; - byte salt[XB_KDF_SALT_SIZE]; - char transition_key_name[TRANSITION_KEY_NAME_MAX_LEN_V2]; - char transition_key_buf[Encryption::KEY_LEN]; - - xb::info() << "Saving " << XTRABACKUP_KEYS_FILE; - - if (my_rand_buffer(salt, sizeof(salt)) != 0) { - return (false); - } - - memset(transition_key_name, 0, sizeof(transition_key_name)); - - if (transition_key == NULL) { - if (!xb_create_transition_key(transition_key_name, transition_key_buf)) { - return (false); - } - transition_key = transition_key_buf; - transition_key_len = Encryption::KEY_LEN; - } - - xb_libgcrypt_init(); - bool ret = xb_derive_key(transition_key, transition_key_len, salt, - sizeof(salt), sizeof(derived_key), derived_key); - - if (!ret) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to derive encryption key."; - return (false); - } - - dberr_t err; - MY_STAT stat_info; - memset(&stat_info, 0, sizeof(MY_STAT)); - - ds_file_t *stream = ds_open(ds_ctxt, XTRABACKUP_KEYS_FILE, &stat_info); - if (stream == NULL) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to create file."; - return (false); - } - - if (ds_write(stream, XTRABACKUP_KEYS_MAGIC_V2, XTRABACKUP_KEYS_MAGIC_SIZE)) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to write magic."; - goto error; - } - - if (ds_write(stream, salt, sizeof(salt))) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to write salt."; - goto error; - } - - if (ds_write(stream, transition_key_name, sizeof(transition_key_name))) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to write transition key name."; - goto error; - } - - err = Fil_space_iterator::for_each_space([&](fil_space_t *space) { - if (space->m_encryption_metadata.m_type == Encryption::NONE) { - return (DB_SUCCESS); - } - if (!xb_tablespace_keys_write_single(stream, derived_key, space->id, - space->m_encryption_metadata.m_key, - space->m_encryption_metadata.m_iv)) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to save tablespace key."; - return (DB_ERROR); - } - return (DB_SUCCESS); - }); - - if (err != DB_SUCCESS) { - goto error; - } - - if (recv_sys->keys != nullptr) { - for (auto &key : *recv_sys->keys) { - if (!xb_tablespace_keys_write_single(stream, derived_key, key.space_id, - key.ptr, key.iv)) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to save tablespace key."; - goto error; - } - } - } - - for (const auto &entry : encryption_info) { - if (!xb_tablespace_keys_write_single(stream, derived_key, entry.first, - entry.second.key, entry.second.iv)) { - xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE - << ": failed to save tablespace key."; - goto error; - } - } - - ds_close(stream); - return (true); - -error: - ds_close(stream); - return (false); -} - /** Read encrypted binlog file header @@ -896,3 +777,157 @@ void xb_keyring_shutdown() { free_list(opt_plugin_load_list_ptr); xtrabackup::components::deinitialize_service_handles(); } + +TablespaceKeyDumper::TablespaceKeyDumper(ds_ctxt_t *ds_ctxt, + const char *transition_key, + size_t transition_key_len) + : m_ds_ctxt(ds_ctxt), + m_transition_key(transition_key), + m_transition_key_len(transition_key_len), + m_stream(nullptr), + m_state(State::Init), + m_finalized(false) { + memset(m_salt, 0, sizeof(m_salt)); + memset(m_derived_key, 0, sizeof(m_derived_key)); + memset(m_transition_key_name, 0, sizeof(m_transition_key_name)); +} + +bool TablespaceKeyDumper::is_initialized() const { + return m_state != State::Init; +} + +bool TablespaceKeyDumper::initialize() { + xb::info() << "Saving " << XTRABACKUP_KEYS_FILE; + + if (my_rand_buffer(m_salt, sizeof(m_salt)) != 0) return false; + + if (m_transition_key == nullptr) { + if (!xb_create_transition_key(m_transition_key_name, + m_transition_key_buf)) { + return false; + } + m_transition_key = m_transition_key_buf; + m_transition_key_len = Encryption::KEY_LEN; + } + + xb_libgcrypt_init(); + if (!xb_derive_key(m_transition_key, m_transition_key_len, m_salt, + sizeof(m_salt), sizeof(m_derived_key), m_derived_key)) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to derive encryption key."; + return false; + } + + MY_STAT stat_info; + memset(&stat_info, 0, sizeof(MY_STAT)); + + m_stream = ds_open(m_ds_ctxt, XTRABACKUP_KEYS_FILE, &stat_info); + if (!m_stream) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to create file."; + return false; + } + + if (ds_write(m_stream, XTRABACKUP_KEYS_MAGIC_V2, + XTRABACKUP_KEYS_MAGIC_SIZE) || + ds_write(m_stream, m_salt, sizeof(m_salt)) || + ds_write(m_stream, m_transition_key_name, + sizeof(m_transition_key_name))) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to write header."; + ds_close(m_stream); + m_stream = nullptr; + return false; + } + + m_state = State::Initialized; + return true; +} + +bool TablespaceKeyDumper::dump_from_spaces(bool use_ddl_tracker) { + if (use_ddl_tracker) { + ut_a(ddl_tracker != nullptr); + } + + if (m_state != State::Initialized && m_state != State::SpacesDumpedOnce) + return fail_state("dump_from_spaces"); + + m_state = State::SpacesDumpedOnce; + + dberr_t err = Fil_space_iterator::for_each_space([&](fil_space_t *space) { + if (space->m_encryption_metadata.m_type == Encryption::NONE) { + return DB_SUCCESS; + } + + if (use_ddl_tracker && ddl_tracker != nullptr) { + if (ddl_tracker->is_tablespace_dropped(space->id)) { + return DB_SUCCESS; + } + } + + if (!xb_tablespace_keys_write_single(m_stream, m_derived_key, space->id, + space->m_encryption_metadata.m_key, + space->m_encryption_metadata.m_iv)) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to save tablespace key."; + return DB_ERROR; + } + + return DB_SUCCESS; + }); + + return err == DB_SUCCESS; +} + +bool TablespaceKeyDumper::dump_from_redo() { + if (m_state != State::SpacesDumpedOnce) return fail_state("dump_from_redo"); + m_state = State::RecoveryDumped; + + if (!recv_sys || !recv_sys->keys) return true; + + for (auto &key : *recv_sys->keys) { + if (!xb_tablespace_keys_write_single(m_stream, m_derived_key, key.space_id, + key.ptr, key.iv)) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to save tablespace key."; + return false; + } + } + + return true; +} + +bool TablespaceKeyDumper::dump_from_encryption_infos() { + if (m_state != State::RecoveryDumped) + return fail_state("dump_from_encryption_infos"); + m_state = State::EncryptionInfosDumped; + + for (const auto &entry : encryption_info) { + if (!xb_tablespace_keys_write_single(m_stream, m_derived_key, entry.first, + entry.second.key, entry.second.iv)) { + xb::error() << "Error writing " << XTRABACKUP_KEYS_FILE + << ": failed to save tablespace key."; + return false; + } + } + + return true; +} + +void TablespaceKeyDumper::finalize() { + if (m_stream) { + ds_close(m_stream); + m_stream = nullptr; + } + m_finalized = true; +} + +TablespaceKeyDumper::~TablespaceKeyDumper() { + assert(m_finalized && "finalize() must be called before destruction"); +} + +bool TablespaceKeyDumper::fail_state(const char *func) { + xb::error() << "Invalid call to " << func + << ": incorrect stage or already called."; + return false; +} diff --git a/storage/innobase/xtrabackup/src/keyring_plugins.h b/storage/innobase/xtrabackup/src/keyring_plugins.h index 93856dce46bc..8aa0438f6e09 100644 --- a/storage/innobase/xtrabackup/src/keyring_plugins.h +++ b/storage/innobase/xtrabackup/src/keyring_plugins.h @@ -22,6 +22,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include "datasink.h" +#include "kdf.h" #include "log0types.h" /** Initialize keyring plugin for backup. Config is read from live mysql server. @@ -58,15 +59,6 @@ bool xb_tablespace_keys_exist(); bool xb_tablespace_keys_load(const char *dir, const char *transition_key, size_t transition_key_len); -/** Dump tablespace keys into encrypted "xtrabackup_keys" file. -@param[in] ds_ctxt datasink context to output file into -@param[in] transition_key transition key used to encrypt - tablespace keys -@param[in] transition_key_len transition key length -@return true if success */ -bool xb_tablespace_keys_dump(ds_ctxt_t *ds_ctxt, const char *transition_key, - size_t transition_key_len); - /** Store binlog password into a backup @@ -97,4 +89,55 @@ variable @param[out] e_m Encryption metadata of redo log @return true on success, else false */ bool xb_load_saved_redo_encryption(Encryption_metadata &em); + +#define TRANSITION_KEY_PREFIX_STR "XBKey" + +const char *const TRANSITION_KEY_PREFIX = TRANSITION_KEY_PREFIX_STR; +const size_t TRANSITION_KEY_PREFIX_LEN = sizeof(TRANSITION_KEY_PREFIX_STR) - 1; +const size_t TRANSITION_KEY_RANDOM_DATA_LEN = 32; +const size_t TRANSITION_KEY_NAME_MAX_LEN_V1 = + Encryption::SERVER_UUID_LEN + 2 + 45; +const size_t TRANSITION_KEY_NAME_MAX_LEN_V2 = + TRANSITION_KEY_PREFIX_LEN + Encryption::SERVER_UUID_LEN + + TRANSITION_KEY_RANDOM_DATA_LEN + 1; + +class TablespaceKeyDumper { + public: + TablespaceKeyDumper(ds_ctxt_t *ds_ctxt, const char *transition_key, + size_t transition_key_len); + + bool initialize(); + bool dump_from_spaces(bool use_ddl_tracker); + bool dump_from_redo(); + bool dump_from_encryption_infos(); + void finalize(); + bool is_initialized() const; + + ~TablespaceKeyDumper(); + + private: + enum class State { + Init, + Initialized, + SpacesDumpedOnce, + RecoveryDumped, + EncryptionInfosDumped + }; + + bool fail_state(const char *func); + + ds_ctxt_t *m_ds_ctxt; + const char *m_transition_key; + size_t m_transition_key_len; + + byte m_derived_key[Encryption::KEY_LEN]; + byte m_salt[XB_KDF_SALT_SIZE]; + char m_transition_key_name[TRANSITION_KEY_NAME_MAX_LEN_V2]; + char m_transition_key_buf[Encryption::KEY_LEN]; + + ds_file_t *m_stream; + State m_state; + bool m_finalized; +}; + #endif // XB_KEYRING_PLUGINS_H diff --git a/storage/innobase/xtrabackup/src/xtrabackup.cc b/storage/innobase/xtrabackup/src/xtrabackup.cc index f61e3ca9f19a..5143e1bf5545 100644 --- a/storage/innobase/xtrabackup/src/xtrabackup.cc +++ b/storage/innobase/xtrabackup/src/xtrabackup.cc @@ -4518,6 +4518,22 @@ void xtrabackup_backup_func(void) { Backup_context backup_ctxt; backup_ctxt.redo_mgr = &redo_mgr; + + TablespaceKeyDumper ts_key_dumper( + ds_data, opt_transition_key, + opt_transition_key != NULL ? strlen(opt_transition_key) : 0); + + if (opt_transition_key != NULL || opt_generate_transition_key) { + if (!ts_key_dumper.initialize()) { + xb::error() << "--transition-key couldn't be initialized. Please check" + << " the transition key, the errors and retry"; + exit(EXIT_FAILURE); + } + backup_ctxt.ts_key_dumper = &ts_key_dumper; + } else { + backup_ctxt.ts_key_dumper = nullptr; + } + if (!backup_start(backup_ctxt)) { exit(EXIT_FAILURE); } @@ -4594,15 +4610,24 @@ void xtrabackup_backup_func(void) { Tablespace_map::instance().serialize(ds_data); - if (opt_transition_key != NULL || opt_generate_transition_key) { - if (!xb_tablespace_keys_dump( - ds_data, opt_transition_key, - opt_transition_key != NULL ? strlen(opt_transition_key) : 0)) { - xb::error() << "failed to dump tablespace keys."; - exit(EXIT_FAILURE); + if (ts_key_dumper.is_initialized()) { + if (!ts_key_dumper.dump_from_spaces(false)) { + xb::error() << "Couldn't dump transition key for all tablespaces."; + } + + if (!ts_key_dumper.dump_from_redo()) { + xb::error() << "Couldn't dump transition key for all tablespaces found" + << " from redo log"; + } + + if (!ts_key_dumper.dump_from_encryption_infos()) { + xb::error() << "Couldn't dump transition key for all tablespaces found" + << " from encryption_info vector "; } } + ts_key_dumper.finalize(); + xtrabackup_destroy_datasinks(); if (wait_throttle) { diff --git a/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys.sh b/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys.sh new file mode 100644 index 000000000000..0b3c141891e6 --- /dev/null +++ b/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys.sh @@ -0,0 +1,30 @@ +# +# PXB-3571 : --transition-key does not save keys with --lock-ddl=reduced +# + + +KEYRING_TYPE="component" +. inc/keyring_common.sh +. inc/keyring_file.sh +configure_server_with_component + +mysql -e "CREATE TABLE t (a INT) ENCRYPTION='y'" test + +shutdown_server +start_server + +for i in {1..100} ; do + mysql -e "INSERT INTO t VALUES ($i)" test +done + +BACKUP_DIR=$topdir/backup +xtrabackup --backup --transition-key=123 --target-dir=$BACKUP_DIR --lock-ddl=reduced +record_db_state test +xtrabackup --prepare --transition-key=123 --target-dir=$BACKUP_DIR --lock-ddl=reduced --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +stop_server +rm -rf $mysql_datadir/* +xtrabackup --copy-back --target-dir=$BACKUP_DIR --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +cp ${instance_local_manifest} $mysql_datadir +cp ${keyring_component_cnf} $mysql_datadir +start_server +verify_db_state test diff --git a/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys_debug.sh b/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys_debug.sh new file mode 100644 index 000000000000..b1108925526b --- /dev/null +++ b/storage/innobase/xtrabackup/test/suites/reducedlock/transition_keys_debug.sh @@ -0,0 +1,67 @@ +# +# PXB-3571 : --transition-key does not save keys with --lock-ddl=reduced +# + +. inc/common.sh +require_debug_pxb_version +require_pro_pxb_version + +KEYRING_TYPE="component" +. inc/keyring_common.sh +. inc/keyring_file.sh +configure_server_with_component + +mysql -e "CREATE TABLE t (a INT) ENCRYPTION='y'" test + +shutdown_server +start_server + +for i in {1..100} ; do + mysql -e "INSERT INTO t VALUES ($i)" test +done + +BACKUP_DIR=$topdir/backup +xtrabackup --backup --transition-key=123 --target-dir=$BACKUP_DIR --lock-ddl=reduced +record_db_state test +xtrabackup --prepare --transition-key=123 --target-dir=$BACKUP_DIR --lock-ddl=reduced --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +stop_server +rm -rf $mysql_datadir/* +xtrabackup --copy-back --target-dir=$BACKUP_DIR --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +cp ${instance_local_manifest} $mysql_datadir +cp ${keyring_component_cnf} $mysql_datadir +start_server +verify_db_state test + +rm -rf $BACKUP_DIR + +mysql -e "CREATE TABLE t2(a INT) ENCRYPTION='y'" test + +xtrabackup --backup --transition-key=123 --target-dir=$BACKUP_DIR --debug-sync="ddl_tracker_before_lock_ddl" --lock-ddl=REDUCED 2> >( tee $topdir/backup.log)& + +job_pid=$! +pid_file=$BACKUP_DIR/xtrabackup_debug_sync +wait_for_xb_to_suspend $pid_file +xb_pid=`cat $pid_file` +echo "backup pid is $job_pid" + +# Generate redo on table than delete it +$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO test.t2 VALUES (1); DROP TABLE test.t2;" test + +# Resume the xtrabackup process +vlog "Resuming xtrabackup" +kill -SIGCONT $xb_pid +run_cmd wait $job_pid + +record_db_state test + +xtrabackup --prepare --transition-key=123 --target-dir=$BACKUP_DIR --lock-ddl=reduced --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +stop_server +rm -rf $mysql_datadir/* +xtrabackup --copy-back --target-dir=$BACKUP_DIR --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args} +cp ${instance_local_manifest} $mysql_datadir +cp ${keyring_component_cnf} $mysql_datadir +start_server +verify_db_state test + +rm -rf $BACKUP_DIR +rm $topdir/backup.log From cc303faa163dd2eb014049fed68ceea224c30724 Mon Sep 17 00:00:00 2001 From: Satya Bodapati Date: Thu, 31 Jul 2025 11:52:26 +0100 Subject: [PATCH 2/2] Fix testcase failures due to format change --- .../suites/reducedlock/external_tablespaces.sh | 14 ++++---------- .../test/suites/reducedlock/parallel_copy.sh | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/storage/innobase/xtrabackup/test/suites/reducedlock/external_tablespaces.sh b/storage/innobase/xtrabackup/test/suites/reducedlock/external_tablespaces.sh index 2d035ae04d1e..5d5273af78a4 100644 --- a/storage/innobase/xtrabackup/test/suites/reducedlock/external_tablespaces.sh +++ b/storage/innobase/xtrabackup/test/suites/reducedlock/external_tablespaces.sh @@ -57,22 +57,18 @@ if ! egrep -q "New undo file: $undo_directory_ext/undo_1.ibu : [0-9]*" $topdir/b die "xtrabackup did not handle new table DDL" fi -if ! egrep -q "Done: Copying file with space_id [0-9]* $undo_directory_ext/undo_1.ibu to $topdir/backup/undo_1.ibu.new" $topdir/backup.log ; then +if ! egrep -q "space_id: [0-9]*, Done: Copying $undo_directory_ext/undo_1.ibu to $topdir/backup/undo_1.ibu.new" $topdir/backup.log ; then die "xtrabackup did not create undo_1.ibu.new file" fi -if ! egrep -q "Done: Copying file with space_id [0-9]* $undo_directory_ext/undo_003.ibu to $topdir/backup/undo_003.ibu.new" $topdir/backup.log ; then +if ! egrep -q "space_id: [0-9]*, Done: Copying $undo_directory_ext/undo_003.ibu to $topdir/backup/undo_003.ibu.new" $topdir/backup.log ; then die "xtrabackup did not create undo_003.ibu.new file" fi -if ! egrep -q "Done: Copying file with space_id [0-9]* $undo_directory_ext/undo_001 to $topdir/backup/undo_001.new" $topdir/backup.log ; then +if ! egrep -q "space_id: [0-9]*, Done: Copying $undo_directory_ext/undo_001 to $topdir/backup/undo_001.new" $topdir/backup.log ; then die "xtrabackup did not create undo_001.new file" fi - - - - mysql -e "SET GLOBAL innodb_purge_rseg_truncate_frequency=default" xtrabackup --prepare --target-dir=$topdir/backup stop_server @@ -85,8 +81,6 @@ stop_server rm -rf $MYSQLD_DATADIR rm -rf $undo_directory_ext - - vlog "case #2: ensure external file-per-table and general tablespaces are handled" data_directory_ext=$TEST_VAR_ROOT/var1/data_dir_ext @@ -144,7 +138,7 @@ if ! egrep -q "DDL tracking : LSN: [0-9]* create space ID: [0-9]* Name: $data_di die "xtrabackup did not handle new table DDL" fi -if ! egrep -q "Done: Copying file with space_id [0-9]* $data_directory_ext/test/OP_DDL_TABLE.ibd to $topdir/backup_new_table/test/OP_DDL_TABLE.ibd.new" $topdir/backup_with_new_table.log ; then +if ! egrep -q "space_id: [0-9]*, Done: Copying $data_directory_ext/test/OP_DDL_TABLE.ibd to $topdir/backup_new_table/test/OP_DDL_TABLE.ibd.new" $topdir/backup_with_new_table.log ; then die "xtrabackup did not create OP_DDL_TABLE.ibd.new file" fi diff --git a/storage/innobase/xtrabackup/test/suites/reducedlock/parallel_copy.sh b/storage/innobase/xtrabackup/test/suites/reducedlock/parallel_copy.sh index 58be0030ab4a..8ec85ddc53a9 100644 --- a/storage/innobase/xtrabackup/test/suites/reducedlock/parallel_copy.sh +++ b/storage/innobase/xtrabackup/test/suites/reducedlock/parallel_copy.sh @@ -31,7 +31,7 @@ kill -SIGCONT $xb_pid run_cmd wait $job_pid # xtrabackup uses an ever incremental thread ID. Second phase copy will start from 20+ -if ! egrep -q '[1-3][0-9] \[Note\] \[MY-[0-9]*\] \[Xtrabackup\] Copying file with space_id [0-9]* test/tb_[0-9]*\.ibd to .*/test/tb_[0-9]*.ibd.new' $topdir/backup.log ; then +if ! egrep -q '[1-3][0-9] \[Note\] \[MY-[0-9]*\] \[Xtrabackup\] space_id: [0-9]*, Copying test/tb_[0-9]*\.ibd to .*/test/tb_[0-9]*.ibd.new' $topdir/backup.log ; then die "xtrabackup did not copied tables in parallel" fi