mbop: add sync_mailbox operation of midb to mbop

References: GXL-630, DESK-3819

Author: grammmike

Makefile.am

@@ -196,7 +196,7 @@ gromox_mbck_SOURCES = tools/mbck.cpp
 gromox_mbck_LDADD = ${libHX_LIBS} ${fmt_LIBS} ${sqlite_LIBS} libgromox_common.la
 gromox_mbop_SOURCES = tools/genimport.cpp tools/genimport.hpp tools/mbop.hpp tools/mbop.cpp
 EXTRA_DIST += tools/mbop_delmsg.cpp tools/mbop_emptyfld.cpp tools/mbop_foreach.cpp tools/mbop_freebusy.cpp tools/mbop_locale.cpp tools/mbop_main.cpp tools/mbop_purge.cpp tools/staticnpmap.cpp
-gromox_mbop_LDADD = ${libHX_LIBS} ${mysql_LIBS} libgromox_common.la libgromox_exrpc.la libgromox_mapi.la libgxs_mysql_adaptor.la
+gromox_mbop_LDADD = ${libHX_LIBS} ${mysql_LIBS} libgromox_common.la libgromox_exrpc.la libgromox_mapi.la libgxs_mysql_adaptor.la libgxs_midb_agent.la
 gromox_mbsize_SOURCES = tools/mbsize.cpp
 gromox_mbsize_LDADD = ${sqlite_LIBS} libgromox_common.la
 gromox_mkmidb_SOURCES = tools/mkmidb.cpp tools/mkshared.cpp tools/mkshared.hpp

doc/changelog.rst

@@ -37,6 +37,7 @@ Enhancements:
 * dscli: try all oxdisco URLs until one succeeds
 * exmdb: support repeated import of permission data (e.g. from kdb2mt)
 * ews: create calendar item after accepting a MR with MacMail
+* mbop: new `sync-midb` subcommand to prebuild midb caches ahead of the first IMAP login
 
 Fixes:
 

doc/gromox-mbop.8

@@ -69,6 +69,8 @@ set\-photo: read user image from stdin and save to store
 set\-websettings, set\-websettings\-persistent, set\-websettings\-recipients:
 read new grommunio-web settings from stdin and save to store
 .IP \(bu 4
+sync\-midb: trigger a midb synchronization run
+.IP \(bu 4
 thaw: unfreeze a mailbox
 .IP \(bu 4
 unload: issue the "unload_store" RPC for a mailbox
@@ -323,6 +325,21 @@ Reads various grommunio-web settings from the store and dumps it to stdout.
 Causes the respective mailbox to be opened by the server. (Any request to the
 information storage server causes the respective mailbox to be opened; and ping
 is technically just a no-op request type.)
+.SH sync\-midb
+.SS Synopsis
+\fBsync-midb\fP [\fB\-f\fP \fIfolder_spec\fP]
+.SS Description
+Sends a request to midb for opening the mailbox and updating the midb-specific
+folder indices, as well as potentially building RFC5322 representations for
+newly-appeared messages. (Once the mailbox is open in midb, it uses
+asynchronous notifications to stay up to date.) Due to implementation details
+of the midb connector, requests will time out after 60 second, even if sync
+procedure inside the midb daemon keeps running in the background.
+.SS Options
+.TP
+\fB\-f\fP \fIfolder_spec\fP
+Forcibly rerun the sync routine for a single specific folder. See section
+"Folder specification". In addition, the special keyword "all" is recognized.
 .SH purge\-datafiles
 The "purge\-datafiles" RPC makes exmdb_provider remove attachment and content
 files from disk that are no longer referenced by any message.

exch/midb/mail_engine.cpp

@@ -3550,12 +3550,20 @@ static int me_xunld(int argc, char **argv, int sockd)
  * Request:
  * 	X-RSYM <store-dir>
  * Response:
- * 	TRUE <0|1|2>
+ * 	TRUE 0: synchro failure
+ * 	TRUE 1: was loaded before (tracking changes live), now is synchronized
+ * 	TRUE 2: was unloaded before (not tracking), now is synchronized
  */
 static int me_xrsym(int argc, char **argv, int sockd)
 {
 	auto idb = me_peek_idb(argv[1]);
 	if (idb == nullptr) {
+		/*
+		 * There is a time between me_peek_idb and me_get_idb that
+		 * another thread may have invoked me_get_idb and already
+		 * triggered an initial sync. Therefore, we are passing `true`
+		 * here to force resync in any case.
+		 */
 		me_get_idb(argv[1], true);
 		return cmd_write(sockd, "TRUE 2\r\n");
 	} else if (!me_sync_mailbox(idb.get(), true)) {

include/gromox/midb_agent.hpp

@@ -46,6 +46,7 @@ extern GX_EXPORT int summary_folder(const char *path, const std::string &folder,
 extern GX_EXPORT int make_folder(const char *path, const std::string &folder, int *perrno);
 extern GX_EXPORT int remove_folder(const char *path, const std::string &folder, int *perrno);
 extern GX_EXPORT int ping_mailbox(const char *path, int *perrno);
+extern GX_EXPORT int sync_mailbox(const char *path, uint64_t folder_id, int *err);
 extern GX_EXPORT int rename_folder(const char *path, const std::string &src_name, const std::string &dst_name, int *perrno);
 extern GX_EXPORT int subscribe_folder(const char *path, const std::string &folder, int *perrno);
 extern GX_EXPORT int unsubscribe_folder(const char *path, const std::string &folder, int *perrno);

mra/midb_agent.cpp

@@ -745,6 +745,33 @@ int ping_mailbox(const char *path, int *perrno)
 	return MIDB_RDWR_ERROR;
 }
 
+int sync_mailbox(const char *path, uint64_t folder_id, int *err)
+{
+	auto conn = get_connection(path);
+	if (conn == nullptr)
+		return MIDB_NO_SERVER;
+	char buf[1024];
+	int len;
+	if (folder_id == 0)
+		len = gx_snprintf(buf, std::size(buf), "X-RSYM %s\r\n", path);
+	else
+		len = gx_snprintf(buf, std::size(buf), "X-RSYF %s %llu\r\n",
+		      path, static_cast<unsigned long long>(folder_id));
+	auto ret = rw_command(conn->sockd, buf, len, std::size(buf));
+	if (ret != 0)
+		return ret;
+	if (strncmp(buf, "TRUE", 4) == 0) {
+		conn.reset();
+		return MIDB_RESULT_OK;
+	} else if (strncmp(buf, "FALSE ", 6) == 0) {
+		conn.reset();
+		if (err != nullptr)
+			*err = strtol(&buf[6], nullptr, 0);
+		return MIDB_RESULT_ERROR;
+	}
+	return MIDB_RDWR_ERROR;
+}
+
 int rename_folder(const char *path, const std::string &src_name,
     const std::string &dst_name, int *perrno)
 {

tools/mbop.hpp

@@ -21,6 +21,7 @@ namespace foreach_wrap { extern int main(int, char **); }
 namespace getfreebusy { extern int main(int, char **); }
 namespace purgesoftdel { extern int main(int, char **); }
 namespace set_locale { extern int main(int, char **); }
+namespace sync_midb { extern int main(int, char **); }
 
 namespace global {
 

tools/mbop_main.cpp

@@ -17,6 +17,7 @@
 #include <gromox/exmdb_rpc.hpp>
 #include <gromox/freebusy.hpp>
 #include <gromox/mapidefs.h>
+#include <gromox/midb_agent.hpp>
 #include <gromox/mysql_adaptor.hpp>
 #include <gromox/process.hpp>
 #include <gromox/svc_loader.hpp>
@@ -211,6 +212,65 @@ static int freeze_main(int argc, char **argv)
 
 }
 
+namespace sync_midb {
+
+static const char *g_folder_spec;
+static constexpr HXoption g_options_table[] = {
+	{{}, 'f', HXTYPE_STRP, &g_folder_spec, {}, {}, {}, "Forcibly rescan this folder", "SPEC"},
+	MBOP_AUTOHELP,
+	HXOPT_TABLEEND,
+};
+
+int main(int argc, char **argv)
+{
+	HXopt6_auto_result argp;
+
+	if (HX_getopt6(g_options_table, argc, argv, nullptr,
+	    HXOPT_USAGEONERR) != HXOPT_ERR_SUCCESS || g_exit_after_optparse)
+		return EXIT_PARAM;
+
+	int err = 0, ret;
+	if (g_folder_spec == nullptr) {
+		ret = midb_agent::ping_mailbox(g_storedir, &err);
+	} else if (strcasecmp(g_folder_spec, "all") == 0) {
+		ret = midb_agent::sync_mailbox(g_storedir, 0, &err);
+	} else {
+		eid_t eid = gi_lookup_eid_by_name(g_storedir, g_folder_spec);
+		if (eid == 0) {
+			mbop_fprintf(stderr, "Not recognized/found: \"%s\"\n", g_folder_spec);
+			return EXIT_FAILURE;
+		}
+		ret = midb_agent::sync_mailbox(g_storedir, eid.gcv(), &err);
+	}
+	switch (ret) {
+	case MIDB_RESULT_OK:
+		if (global::g_verbose_mode)
+			printf("midb sync triggered\n");
+		return EXIT_SUCCESS;
+	case MIDB_RESULT_ERROR:
+		mbop_fprintf(stderr, "midb sync failed: backend returned %d\n", err);
+		break;
+	case MIDB_NO_SERVER:
+		mbop_fprintf(stderr, "midb sync failed: midb backend unavailable\n");
+		break;
+	case MIDB_RDWR_ERROR:
+		mbop_fprintf(stderr, "midb sync failed: protocol error while talking to midb\n");
+		break;
+	case MIDB_LOCAL_ENOMEM:
+		mbop_fprintf(stderr, "midb sync failed: local memory exhausted\n");
+		break;
+	case MIDB_TOO_MANY_RESULTS:
+		mbop_fprintf(stderr, "midb sync failed: backend returned too much data\n");
+		break;
+	default:
+		mbop_fprintf(stderr, "midb sync failed: unexpected midb agent status %d\n", ret);
+		break;
+	}
+	return EXIT_FAILURE;
+}
+
+}
+
 static errno_t resolvename(const GUID &guid, const char *name, bool create,
     uint16_t *out)
 {
@@ -426,8 +486,10 @@ static int single_user_wrap(int argc, char **argv)
 	return ret;
 }
 
-static constexpr generic_module g_dfl_svc_plugins[] =
-	{{"libgxs_mysql_adaptor.so", SVC_mysql_adaptor}};
+static constexpr generic_module g_dfl_svc_plugins[] = {
+	{"libgxs_mysql_adaptor.so", SVC_mysql_adaptor},
+	{"libgxs_midb_agent.so", SVC_midb_agent},
+};
 
 int main(int argc, char **argv)
 {
@@ -447,7 +509,7 @@ int main(int argc, char **argv)
 	argv = result.uarg;
 	if (argc == 0)
 		return global::help();
-	service_init({nullptr, g_dfl_svc_plugins, 1});
+	service_init({nullptr, g_dfl_svc_plugins, 2});
 	auto cl_1 = HX::make_scope_exit(service_stop);
 	if (service_run_early() != 0 || service_run() != 0) {
 		fprintf(stderr, "service_run: failed\n");
@@ -535,6 +597,8 @@ int cmd_parser(int argc, char **argv)
 		return set_locale::main(argc, argv);
 	else if (strcmp(argv[0], "get-freebusy") == 0 || strcmp(argv[0], "gfb") == 0)
 		return getfreebusy::main(argc, argv);
+	else if (strcmp(argv[0], "sync-midb") == 0)
+		return sync_midb::main(argc, argv);
 
 	if (strcmp(argv[0], "clear-profile") == 0) {
 		auto ret = delstoreprop(argc, argv, PSETID_Gromox, "zcore_profsect", PT_BINARY);