exmdb: add write_message_v3 EXRPC

The deliver_message RPC has a "digest" parameter with which a process
(gromox-delivery) can indicate to the Information Store (exmdb) that
an RFC5322 Internet Mail file is logically linked to a newly-created
MAPI message.

The write_message RPC is the "not delivery, just placement" companion
function, which is used by the import tools. Having a digest
parameter is equally valuable. This commit adds the parameter and its
handling.

References: GXH-155, GXF-2229

Author: jengelh

exch/emsmdb/fastupctx_object.cpp

@@ -213,8 +213,10 @@ fastupctx_object_write_message(fastupctx_object *pctx, uint64_t folder_id)
 		return err;
 	auto pinfo = emsmdb_interface_get_emsmdb_info();
 	ec_error_t e_result = ecRpcFailed;
+	uint64_t outmid = 0, outcn = 0;
 	if (!exmdb_client->write_message(dir, pinfo->cpid, folder_id,
-	    pctx->m_content, &e_result) || e_result != ecSuccess)
+	    pctx->m_content, {}, &outmid, &outcn, &e_result) ||
+	    e_result != ecSuccess)
 		return e_result;
 	return ecSuccess;
 }

exch/ews/context.cpp

@@ -499,8 +499,11 @@ sItem EWSContext::create(const std::string& dir, const sFolderSpec& parent, cons
 	auto messageId = content.proplist.get<const uint64_t>(PidTagMid);
 	if (!messageId)
 		throw DispatchError(E3112);
+	if (!content.proplist.has(PidTagChangeNumber))
+		throw EWSError::ItemSave(E3254);
+	uint64_t outmid = 0, outcn = 0;
 	if (!m_plugin.exmdb.write_message(dir.c_str(), CP_ACP, parent.folderId,
-	    &content, &error) || error != ecSuccess)
+	    &content, {}, &outmid, &outcn, &error) || error != ecSuccess)
 		throw EWSError::ItemSave(E3254);
 
 	sShape retshape = sShape(tItemResponseShape());

exch/ews/requests.cpp

@@ -1644,9 +1644,12 @@ void process(mUpdateItemRequest&& request, XMLElement* response, const EWSContex
 			auto error = content->proplist.set(PidTagMid, EWSContext::construct<uint64_t>(rop_util_make_eid(1, mid.message_global_counter)));
 			if (error == ecServerOOM)
 				throw EWSError::ItemSave(E3035);
+			if (!content->proplist.has(PidTagChangeNumber))
+				throw EWSError::ItemSave(E3255);
+			uint64_t outmid = 0, outcn = 0;
 			if (!ctx.plugin().exmdb.write_message(dir.c_str(),
-			    CP_ACP, parentFolder.folderId, content.get(),
-			    &error) || error != ecSuccess)
+			    CP_ACP, parentFolder.folderId, content.get(), {},
+			    &outmid, &outcn, &error) || error != ecSuccess)
 				throw EWSError::ItemSave(E3255);
 		} else {
 			TPROPVAL_ARRAY props = shape.write();

exch/exmdb/instance.cpp

@@ -1513,8 +1513,9 @@ BOOL exmdb_server::flush_instance(const char *dir, uint32_t instance_id,
 	dbase.reset();
 	pdb.reset();
 	g_inside_flush_instance = true;
-	BOOL b_result = exmdb_server::write_message(dir, CP_ACP,
-	                folder_id, pmsgctnt, pe_result);
+	uint64_t outmid = 0, outcn = 0;
+	BOOL b_result = exmdb_server::write_message(dir, CP_ACP, folder_id,
+	                pmsgctnt, {}, &outmid, &outcn, pe_result);
 	g_inside_flush_instance = false;
 	exmdb_server::set_public_username(nullptr);
 	return b_result;

exch/exmdb/message.cpp

@@ -3822,8 +3822,9 @@ BOOL exmdb_server::deliver_message(const char *dir, const char *from_address,
  *   replaced.
  * - If PR_LAST_MODIFICATION_TIME is not present, it will be set to now().
  */
-BOOL exmdb_server::write_message_v2(const char *dir, cpid_t cpid,
+BOOL exmdb_server::write_message(const char *dir, cpid_t cpid,
     uint64_t folder_id, const MESSAGE_CONTENT *pmsgctnt,
+    const std::string &digest_stream,
     uint64_t *outmid, uint64_t *outcn, ec_error_t *pe_result)
 {
 	BOOL b_exist = false;
@@ -3868,6 +3869,26 @@ BOOL exmdb_server::write_message_v2(const char *dir, cpid_t cpid,
 		return false;
 	}
 
+	if (digest_stream.size() > 0) {
+		Json::Value digest;
+		std::string mid_string;
+		if (json_from_str(digest_stream, digest) &&
+		    digest["file"].asString().size() > 0) {
+			std::string ext_file = exmdb_server::get_dir() + "/ext/"s + digest["file"].asString();
+			wrapfd fd = open(ext_file.c_str(), O_CREAT | O_TRUNC | O_WRONLY, FMODE_PRIVATE);
+			if (fd.get() >= 0) {
+				if (HXio_fullwrite(fd.get(), digest_stream.c_str(), digest_stream.size()) < 0 ||
+				    fd.close_wr() != 0) {
+					mlog(LV_ERR, "E-1319: write %s: %s", ext_file.c_str(), strerror(errno));
+					return false;
+				}
+				if (!common_util_set_mid_string(pdb->psqlite,
+				    *outmid, digest["file"].asCString()))
+					return false;
+			}
+		}
+	}
+
 	auto dbase = pdb->lock_base_wr();
 	db_conn::NOTIFQ notifq;
 	if (b_exist) {
@@ -3886,18 +3907,6 @@ BOOL exmdb_server::write_message_v2(const char *dir, cpid_t cpid,
 	return TRUE;
 }
 
-BOOL exmdb_server::write_message(const char *dir, cpid_t cpid,
-    uint64_t folder_id, const MESSAGE_CONTENT *ctnt, ec_error_t *e_result)
-{
-	if (!ctnt->proplist.has(PidTagChangeNumber)) {
-		*e_result = ecRpcFailed;
-		return TRUE;
-	}
-	uint64_t outmid = 0, outcn = 0;
-	return write_message_v2(dir, cpid, folder_id, ctnt,
-	       &outmid, &outcn, e_result);
-}
-
 /**
  * @username:   Used for adjusting public store readstates
  */

exch/exmdb/names.cpp

@@ -119,7 +119,7 @@ static constexpr const char *exmdb_rpc_names[] = {
 	E(empty_folder_rule),
 	E(update_folder_rule),
 	"deliver_message_v1",
-	E(write_message),
+	"write_message_v1",
 	E(read_message),
 	E(get_content_sync),
 	E(get_hierarchy_sync),
@@ -150,11 +150,12 @@ static constexpr const char *exmdb_rpc_names[] = {
 	E(recalc_store_size),
 	E(movecopy_folder),
 	E(create_folder),
-	E(write_message_v2),
+	"write_message_v2",
 	E(imapfile_read),
 	E(imapfile_write),
 	E(imapfile_delete),
 	E(cgkreset),
+	E(write_message),
 };
 #undef E
 
@@ -163,7 +164,7 @@ namespace exmdb {
 const char *exmdb_rpc_idtoname(exmdb_callid i)
 {
 	auto j = static_cast<uint8_t>(i);
-	static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::cgkreset) + 1);
+	static_assert(std::size(exmdb_rpc_names) == static_cast<uint8_t>(exmdb_callid::write_message) + 1);
 	auto s = j < std::size(exmdb_rpc_names) ? exmdb_rpc_names[j] : nullptr;
 	return znul(s);
 }

exch/midb/mail_engine.cpp

@@ -2239,9 +2239,10 @@ static int me_minst(int argc, char **argv, int sockd) try
 	if (cpid == CP_ACP)
 		cpid = static_cast<cpid_t>(1252);
 	ec_error_t e_result = ecRpcFailed;
+	uint64_t outmid = 0, outcn = 0;
 	if (!exmdb_client->write_message(argv[1], cpid,
-	    rop_util_make_eid_ex(1, folder_id), pmsgctnt, &e_result) ||
-	    e_result != ecSuccess)
+	    rop_util_make_eid_ex(1, folder_id), pmsgctnt, {},
+	    &outmid, &outcn, &e_result) || e_result != ecSuccess)
 		return MIDB_E_MDB_WRITEMESSAGE;
 	return cmd_write(sockd, "TRUE\r\n");
 } catch (const std::bad_alloc &) {

exch/zcore/common_util.cpp

@@ -1502,8 +1502,10 @@ ec_error_t cu_remote_copy_message(store_object *src_store, uint64_t message_id,
 	if (err != ecSuccess)
 		return err;
 	err = ecError;
+	uint64_t outmid = 0, outcn = 0;
 	if (!exmdb_client->write_message(dst_store->get_dir(), pinfo->cpid,
-	    folder_id1, pmsgctnt, &err) || err != ecSuccess)
+	    folder_id1, pmsgctnt, {}, &outmid, &outcn, &err) ||
+	    err != ecSuccess)
 		return err;
 	return ecSuccess;
 }

include/gromox/exmdb_idef.hpp

@@ -117,8 +117,7 @@ EXMIDL(update_folder_permission, (const char *dir, uint64_t folder_id, BOOL b_fr
 EXMIDL(empty_folder_rule, (const char *dir, uint64_t folder_id))
 EXMIDL(update_folder_rule, (const char *dir, uint64_t folder_id, uint16_t count, const RULE_DATA *prow, IDLOUT BOOL *b_exceed))
 EXMIDL(deliver_message, (const char *dir, const char *from_address, const char *account, cpid_t cpid, uint32_t dlflags, const MESSAGE_CONTENT *pmsg, const char *pdigest, IDLOUT uint64_t *folder_id, uint64_t *message_id, uint32_t *result))
-EXMIDL(write_message, (const char *dir, cpid_t cpid, uint64_t folder_id, const MESSAGE_CONTENT *pmsgctnt, IDLOUT ec_error_t *e_result))
-EXMIDL(write_message_v2, (const char *dir, cpid_t cpid, uint64_t folder_id, const MESSAGE_CONTENT *pmsgctnt, IDLOUT uint64_t *outmid, uint64_t *outcn, ec_error_t *e_result))
+EXMIDL(write_message, (const char *dir, cpid_t cpid, uint64_t folder_id, const MESSAGE_CONTENT *pmsgctnt, const std::string &digest, IDLOUT uint64_t *outmid, uint64_t *outcn, ec_error_t *e_result))
 EXMIDL(read_message, (const char *dir, const char *username, cpid_t cpid, uint64_t message_id, IDLOUT MESSAGE_CONTENT **pmsgctnt))
 EXMIDL(get_content_sync, (const char *dir, uint64_t folder_id, const char *username, const idset *pgiven, const idset *pseen, const idset *pseen_fai, const idset *pread, cpid_t cpid, const RESTRICTION *prestriction, BOOL b_ordered, IDLOUT uint32_t *fai_count, uint64_t *fai_total, uint32_t *normal_count, uint64_t *normal_total, EID_ARRAY *updated_mids, EID_ARRAY *chg_mids, uint64_t *last_cn, EID_ARRAY *given_mids, EID_ARRAY *deleted_mids, EID_ARRAY *nolonger_mids, EID_ARRAY *read_mids, EID_ARRAY *unread_mids, uint64_t *last_readcn))
 EXMIDL(get_hierarchy_sync, (const char *dir, uint64_t folder_id, const char *username, const idset *pgiven, const idset *pseen, IDLOUT FOLDER_CHANGES *fldchgs, uint64_t *last_cn, EID_ARRAY *given_fids, EID_ARRAY *deleted_fids))

include/gromox/exmdb_rpc.hpp

@@ -135,7 +135,7 @@ enum class exmdb_callid : uint8_t {
 	empty_folder_rule = 0x6b,
 	update_folder_rule = 0x6c,
 	// deliver_message_v1 = 0x6d,
-	write_message = 0x6e,
+	// write_message_v1 = 0x6e,
 	read_message = 0x6f,
 	get_content_sync = 0x70,
 	get_hierarchy_sync = 0x71,
@@ -166,11 +166,12 @@ enum class exmdb_callid : uint8_t {
 	recalc_store_size = 0x8a,
 	movecopy_folder = 0x8b,
 	create_folder = 0x8c,
-	write_message_v2 = 0x8d,
+	// write_message_v2 = 0x8d,
 	imapfile_read = 0x8e,
 	imapfile_write = 0x8f,
 	imapfile_delete = 0x90,
 	cgkreset = 0x91,
+	write_message /* v3 */ = 0x92,
 	/* update exch/exmdb/names.cpp:exmdb_rpc_idtoname! */
 };
 
@@ -770,11 +771,11 @@ struct exreq_deliver_message final : public exreq {
 };
 
 struct exreq_write_message final : public exreq {
-	cpid_t cpid;
-	uint64_t folder_id;
-	MESSAGE_CONTENT *pmsgctnt;
+	cpid_t cpid{};
+	uint64_t folder_id = 0;
+	MESSAGE_CONTENT *pmsgctnt = nullptr;
+	std::string digest;
 };
-using exreq_write_message_v2 = exreq_write_message;
 
 struct exreq_read_message final : public exreq {
 	char *username;
@@ -1316,7 +1317,7 @@ struct exresp_autoreply_tsquery final : public exresp {
 	uint64_t tdiff = 0;
 };
 
-struct exresp_write_message_v2 final : public exresp {
+struct exresp_write_message final : public exresp {
 	uint64_t outmid = 0, outcn = 0;
 	ec_error_t e_result{};
 };
@@ -1367,7 +1368,6 @@ using exresp_purge_datafiles = exresp;
 using exresp_autoreply_tsupdate = exresp;
 using exresp_recalc_store_size = exresp;
 using exresp_flush_instance = exresp_error;
-using exresp_write_message = exresp_error;
 using exresp_movecopy_folder = exresp_error;
 using exresp_imapfile_write = exresp;
 using exresp_imapfile_delete = exresp;