exmdb: add cgkreset functionality

exchange.sqlite3 stores CNs not only in the PR_CHANGE_KEY property,
but also the folders.change_number and messages.change_number SQL
column. cgkrepair (as a Gromox/MAPI client) cannot reach that. This
makes a server operation necessary.

cgkreset is this new function. In contrast to cgkrepair, it does not
try to determine whether objects need fixing; it just bumps
everything for simplicity of implementation.

Author: jengelh

doc/gromox-mbop.8

@@ -24,6 +24,8 @@ store of a domain, leave out the local part, i.e. use
 \fB(\fP command1 c1args \fB) (\fP command2 c2args \fB)\fP: command
 chaining
 .IP \(bu 4
+cgkreset: reset synchronization state (PR_CHANGE_KEY, PR_PREDECESSOR_LIST)
+.IP \(bu 4
 clear\-photo: delete user picture
 .IP \(bu 4
 clear\-profile: delete user's PHP-MAPI profile
@@ -82,6 +84,22 @@ time around and \(...\) does not cache the input for any subcommands.
 .SS Examples
 Run two commands for a user: gromox\-mbop \-u [email protected] \\( purge\-softdelete -r /
 \\) \\( purge\-datafiles \\)
+.SH cgkreset
+cgkreset resets Change Numbers on all folder and message objects, PR_CHANGE_KEY
+and PR_PREDECESSOR_LIST values. The use cases for cgkreset are:
+.IP \(bu 4
+when the mailbox has CN corruption and Incremental Change Synchronization (by
+e.g. Outlook or grommunio-sync) is hampered (e.g. message flags/color updates
+not transferred)
+.IP \(bu 4
+when the mailbox has CN corruption and gromox-http/emsmdb has thrown the error
+"INSERT INTO messages ... UNIQUE constraint failed: messages.change_number"
+.PP
+After execution, .ost files referencing the reset mailbox should be deleted.
+`gromox\-mbop cgkreset` is different from `/usr/libexec/gromox/cgkrepair` in
+that cgkreset does not make any attempts to keep synchronization state; it just
+resets everything, unconditionally. On the other hand, cgkrepair is unable to
+get to all fields that may need resetting.
 .SH clear\-photo
 The clear\-photo command will delete the user picture. Note that, when there is
 no mailbox-level profile picture set, Gromox server processes may serve an

exch/exmdb/db_engine.cpp

@@ -285,6 +285,158 @@ BOOL db_engine_unload_db(const char *path)
 	return FALSE;
 }
 
+/**
+ * @db:      sqlite handle
+ * @last_cn: counter for most recently assigned CN (GCV)
+ * @q_list:  query for obtaining some objects
+ *           (shall return 2 columns; id and eligibility)
+ * @q_cn:    query for updating per-object CN
+ * @q_prop:  query for updating per-object CK & PCL
+ *
+ * Iterate over an object set and assign new Change Numbers, Change Keys and
+ * Predecessor Change Lists.
+ */
+static bool cgkreset_3(sqlite3 *db, uint64_t &last_cn, const GUID &store_guid,
+    const char *q_list, const char *q_cn, const char *q_prop)
+{
+	std::vector<std::pair<uint64_t, uint64_t>> gcv_list;
+	auto stm = gx_sql_prep(db, q_list);
+	if (stm == nullptr)
+		return false;
+	while (stm.step() == SQLITE_ROW)
+		gcv_list.emplace_back(stm.col_uint64(0), stm.col_uint64(1));
+	stm = gx_sql_prep(db, q_cn);
+	if (stm == nullptr)
+		return false;
+	auto stm_prop = gx_sql_prep(db, q_prop);
+	if (stm_prop == nullptr)
+		return false;
+
+	for (auto [objid, parent_attid] : gcv_list) {
+		auto next_cn = last_cn + 1;
+		stm.reset();
+		stm.bind_int64(1, next_cn);
+		stm.bind_int64(2, objid);
+		if (stm.step() != SQLITE_DONE)
+			return false;
+		++last_cn;
+		if (parent_attid != 0)
+			/* CK/PCL on attachments makes no sense */
+			continue;
+
+		char buf[23];
+		XID xid{store_guid, rop_util_make_eid_ex(1, last_cn)};
+		EXT_PUSH ep;
+		if (!ep.init(&buf[1], sizeof(buf) - 1, 0))
+			return false;
+		ep.p_xid(xid);
+		stm_prop.reset();
+		stm_prop.bind_blob(1, &buf[1], ep.m_offset);
+		stm_prop.bind_int64(2, objid);
+		stm_prop.bind_int64(3, PR_CHANGE_KEY);
+		if (stm_prop.step() != SQLITE_DONE)
+			return false;
+
+		stm_prop.reset();
+		buf[0] = 22;
+		stm_prop.bind_blob(1, buf, 23);
+		stm_prop.bind_int64(2, objid);
+		stm_prop.bind_int64(3, PR_PREDECESSOR_CHANGE_LIST);
+		if (stm_prop.step() != SQLITE_DONE)
+			return false;
+	}
+	return true;
+}
+
+static bool cgkreset_2(sqlite3 *db, uint64_t &last_cn, const GUID &store_guid,
+    unsigned int flags)
+{
+	if (flags & CGKRESET_FOLDERS) {
+		auto succ = cgkreset_3(db, last_cn, store_guid,
+		            "SELECT folder_id, NULL FROM folders",
+		            "UPDATE folders SET change_number=? WHERE folder_id=?",
+		            "UPDATE folder_properties SET propval=? WHERE folder_id=? AND proptag=?");
+		if (!succ)
+			return false;
+	}
+	if (flags & CGKRESET_MESSAGES) {
+		auto succ = cgkreset_3(db, last_cn, store_guid,
+		            "SELECT message_id, parent_attid FROM messages",
+		            "UPDATE messages SET change_number=? WHERE message_id=?",
+		            "UPDATE message_properties SET propval=? WHERE message_id=? AND proptag=?");
+		if (!succ)
+			return false;
+	}
+	return true;
+}
+
+/**
+ * Obtain essential parameters for a global CN/CK reassignment. In doing so, it
+ * performs the first sanity check and returns a bumped last_cn if necessary.
+ */
+static bool cgkreset_load_param(sqlite3 *db, uint64_t &last_cn, GUID &store_guid)
+{
+	auto stm = gx_sql_prep(db, "SELECT config_value FROM configurations WHERE config_id=?");
+	if (stm == nullptr)
+		return false;
+	stm.bind_int64(1, CONFIG_ID_MAILBOX_GUID);
+	if (stm.step() != SQLITE_ROW)
+		return false;
+	store_guid.from_str(stm.col_text(0));
+	stm.reset();
+	stm.bind_int64(1, CONFIG_ID_LAST_CHANGE_NUMBER);
+	if (stm.step() != SQLITE_ROW)
+		return false;
+	last_cn = stm.col_uint64(0);
+
+	stm = gx_sql_prep(db, "SELECT MAX(change_number) FROM folders");
+	if (stm == nullptr)
+		return false;
+	if (stm.step() == SQLITE_ROW)
+		last_cn = std::max(last_cn, stm.col_uint64(0));
+	stm = gx_sql_prep(db, "SELECT MAX(change_number) FROM messages");
+	if (stm == nullptr)
+		return false;
+	if (stm.step() == SQLITE_ROW)
+		last_cn = std::max(last_cn, stm.col_uint64(0));
+	return true;
+}
+
+static bool cgkreset_save_param(sqlite3 *db, uint64_t last_cn)
+{
+	auto stm = gx_sql_prep(db, "UPDATE configurations SET config_value=? WHERE config_id=?");
+	if (stm == nullptr)
+		return false;
+	stm.bind_int64(1, last_cn);
+	stm.bind_int64(2, CONFIG_ID_LAST_CHANGE_NUMBER);
+	return stm.step() == SQLITE_DONE;
+}
+
+BOOL db_engine_cgkreset(const char *dir, uint32_t flags)
+{
+	auto db = db_engine_get_db(dir);
+	if (!db)
+		return false;
+	auto xact = gx_sql_begin(db->psqlite, txn_mode::write);
+	if (!xact)
+		return false;
+	uint64_t last_cn = 0;
+	GUID store_guid;
+	if (!cgkreset_load_param(db->psqlite, last_cn, store_guid))
+		return false;
+	if (flags & CGKRESET_ZERO_LASTCN)
+		last_cn = 0;
+	if (flags & (CGKRESET_ZERO_LASTCN | CGKRESET_FOLDERS | CGKRESET_MESSAGES)) {
+		auto succ = cgkreset_2(db->psqlite, last_cn, store_guid, flags);
+		if (!succ)
+			return false;
+	}
+	auto succ = cgkreset_save_param(db->psqlite, last_cn);
+	if (!succ)
+		return false;
+	return xact.commit() == SQLITE_OK;
+}
+
 dynamic_node::dynamic_node(dynamic_node &&o) noexcept :
 	folder_id(o.folder_id), search_flags(o.search_flags),
 	prestriction(o.prestriction), folder_ids(o.folder_ids)

exch/exmdb/db_engine.hpp

@@ -221,6 +221,7 @@ extern void db_engine_stop();
 
 extern db_conn_ptr db_engine_get_db(const char *dir);
 extern BOOL db_engine_vacuum(const char *path);
+extern BOOL db_engine_cgkreset(const char *dir, uint32_t flags);
 BOOL db_engine_unload_db(const char *path);
 extern BOOL db_engine_enqueue_populating_criteria(const char *dir, cpid_t, uint64_t folder_id, BOOL recursive, const RESTRICTION *, const LONGLONG_ARRAY *folder_ids);
 extern bool db_engine_check_populating(const char *dir, uint64_t folder_id);

exch/exmdb/names.cpp

@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later, OR GPL-2.0-or-later WITH linking exception
+// SPDX-License-Identifier: AGPL-3.0-or-later
 // SPDX-FileCopyrightText: 2025 grommunio GmbH
 // This file is part of Gromox.
 #include <gromox/defs.h>
@@ -154,6 +154,7 @@ static constexpr const char *exmdb_rpc_names[] = {
 	E(imapfile_read),
 	E(imapfile_write),
 	E(imapfile_delete),
+	E(cgkreset),
 };
 #undef E
 
@@ -162,7 +163,7 @@ namespace exmdb {
 const char *exmdb_rpc_idtoname(exmdb_callid i)
 {
 	auto j = static_cast<uint8_t>(i);
-	static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::imapfile_delete) + 1);
+	static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::cgkreset) + 1);
 	auto s = j < std::size(exmdb_rpc_names) ? exmdb_rpc_names[j] : nullptr;
 	return znul(s);
 }

exch/exmdb/store2.cpp

@@ -53,6 +53,11 @@ BOOL exmdb_server::unload_store(const char *dir)
 	return db_engine_unload_db(dir);
 }
 
+BOOL exmdb_server::cgkreset(const char *dir, uint32_t flags)
+{
+	return db_engine_cgkreset(dir, flags);
+}
+
 BOOL exmdb_server::notify_new_mail(const char *dir, uint64_t folder_id,
 	uint64_t message_id)
 {

include/gromox/exmdb_idef.hpp

@@ -140,3 +140,4 @@ EXMIDL(recalc_store_size, (const char *dir, uint32_t flags))
 EXMIDL(imapfile_read, (const char *dir, const std::string &type, const std::string &mid, IDLOUT std::string *data))
 EXMIDL(imapfile_write, (const char *dir, const std::string &type, const std::string &mid, const std::string &data))
 EXMIDL(imapfile_delete, (const char *dir, const std::string &type, const std::string &mid))
+EXMIDL(cgkreset, (const char *dir, uint32_t flags))

include/gromox/exmdb_rpc.hpp

@@ -170,7 +170,8 @@ enum class exmdb_callid : uint8_t {
 	imapfile_read = 0x8e,
 	imapfile_write = 0x8f,
 	imapfile_delete = 0x90,
-	/* update exch/exmdb_provider/names.cpp:exmdb_rpc_idtoname! */
+	cgkreset = 0x91,
+	/* update exch/exmdb/names.cpp:exmdb_rpc_idtoname! */
 };
 
 struct exreq {
@@ -863,7 +864,19 @@ struct exreq_imapfile_write final : public exreq {
 	std::string type, mid, data;
 };
 
+/**
+ * FOLDERS:     process folders
+ * MESSAGES:    process messages
+ * ZERO_LASTCN: reset all CNs, start from 0 (implies FOLDERS|MESSAGES)
+ */
+enum cgkreset_flags {
+	CGKRESET_FOLDERS     = 0x1U,
+	CGKRESET_MESSAGES    = 0x2U,
+	CGKRESET_ZERO_LASTCN = 0x4U,
+};
+
 using exreq_imapfile_delete = exreq_imapfile_read;
+using exreq_cgkreset = exreq_recalc_store_size;
 
 struct exresp {
 	exresp() = default; /* Prevent use of direct-init-list */
@@ -1358,6 +1371,7 @@ using exresp_write_message = exresp_error;
 using exresp_movecopy_folder = exresp_error;
 using exresp_imapfile_write = exresp;
 using exresp_imapfile_delete = exresp;
+using exresp_cgkreset = exresp;
 
 struct DB_NOTIFY_DATAGRAM {
 	char *dir = nullptr;

lib/exmdb_ext.cpp

@@ -2409,7 +2409,8 @@ static pack_result exmdb_push(EXT_PUSH &x, const exreq_imapfile_write &d)
 	E(write_message_v2) \
 	E(imapfile_read) \
 	E(imapfile_write) \
-	E(imapfile_delete)
+	E(imapfile_delete) \
+	E(cgkreset)
 
 /**
  * This uses *& because we do not know which request type we are going to get
@@ -3744,7 +3745,8 @@ static pack_result exmdb_push(EXT_PUSH &x, const exresp_imapfile_read &d)
 	E(autoreply_tsupdate) \
 	E(recalc_store_size) \
 	E(imapfile_write) \
-	E(imapfile_delete)
+	E(imapfile_delete) \
+	E(cgkreset)
 #define RSP_WITH_ARGS \
 	E(get_all_named_propids) \
 	E(get_named_propids) \

tools/mbop.hpp

@@ -13,6 +13,7 @@ using LLU = unsigned long long;
 
 static constexpr int EXIT_PARAM = 2;
 
+namespace cgkreset { extern int main(int, char **); }
 namespace delmsg { extern int main(int, char **); }
 namespace emptyfld { extern int main(int, char **); }
 namespace foreach_wrap { extern int main(int, char **); }

tools/mbop_main.cpp

@@ -76,7 +76,7 @@ static constexpr HXoption g_options_table[] = {
 
 void command_overview()
 {
-	fprintf(stderr, "Commands:\n\tclear-photo clear-profile clear-rwz delmsg "
+	fprintf(stderr, "Commands:\n\tcgkreset clear-photo clear-profile clear-rwz delmsg "
 		"echo-maildir echo-username "
 		"emptyfld get-freebusy get-photo get-websettings "
 		"get-websettings-persistent "
@@ -514,6 +514,8 @@ int cmd_parser(int argc, char **argv)
 	} else if (strcmp(argv[0], "echo-username") == 0) {
 		printf("%s\n", g_dstuser.c_str());
 		return EXIT_SUCCESS;
+	} else if (strcmp(argv[0], "cgkreset") == 0) {
+		return cgkreset::main(argc, argv);
 	}
 	return simple_rpc::main(argc, argv);
 }