exmdb: cure double notification of content table additions

Under these conditions combined:

* MAPI client has a content table open on a folder
* The MAPI content table has a sort order defined
  (this causes a UNIQUE index to be added to the sqlite table)
* Message delivery executes an OP_MOVE rule to the same folder
* Twostep Rule Processor is used

then the following log message is observed:

```
sqlite3_exec(tables.sqlite3) "INSERT INTO t8 (inst_id, prev_id, row_type,
depth, inst_num, idx) VALUES (2097156, 1, 2, 0, 0, 2)": UNIQUE constraint
failed: t8.inst_id (19) at [...]"
```

maps to

[/usr/src/debug/gromox-2.46.20.bac7ae5-lp156.2.1.x86_64
exch/exmdb/db_engine.hpp:208
exch/exmdb/db_engine.cpp:2122 void db_conn::notify_new_mail last line
exch/exmdb/store2.cpp:68 (discriminator 6)
include/exmdb_dispatch.cpp:962
]

When a *categorized* sort order is set, the message is slightly
different:

```
sqlite3_step(tables.sqlite3) "INSERT INTO t29 (inst_id, row_type,
row_stat, parent_id, depth, count, unread, inst_num, value, extremum,
prev_id) VALUES (2097158, 2, NULL, 1, 1, NULL, NULL, 0, NULL, 0,
-1)": UNIQUE constraint failed: t29.inst_id (19)
```

(no backtrace)

cttbl_add_row is called from both notify_new_mail and
notify_message_movecopy.

[Onestep Rule Processor does not exhibit the problem because it uses
cu_copy_message rather than exmdb_server::movecopy_messages, not
generating notifications.]

The fix here appears to be to use the transport_new_mail EXRPC
instead of notify_new_mail, to eschew the extra call to
cttbl_add_row. In addition, deliver_message (which runs in any case)
should emit the message_created notification ahead of the OP_MOVE.

References: DESK-3519

Author: jengelh

doc/changelog.rst

@@ -20,7 +20,9 @@ Fixes:
 * midb, imap: make EXPUNGE synchronous so that old UIDs don't reappear in
   a subsequent FETCH
 * midb: respect setting \Deleted, \Answered, \Flagged during APPEND
-
+* exmdb: resolve a case of "INSERT INTO t... UNIQUE constraint failed" log
+  message appearing when the Twostep Rule Processor and a Move rule is
+  encountered and a MAPI client has a Content Table with Sort Order open.
 
 
 Gromox 2.46 (2025-05-28)

exch/exmdb/db_engine.cpp

@@ -553,7 +553,7 @@ void db_base::get_dbs(const char* dir, sqlite3 *&main, sqlite3 *&eph)
  * @brief      Initialize and unlock database
  *
  * Perform database initializations that should not be run for each connection:
- * - Remove residue ephemeral db
+ * - Remove residual ephemeral tables db from last process
  * - Perform schema upgrade
  *
  * @param      dir     User or domain base directory

exch/exmdb/message.cpp

@@ -2929,9 +2929,9 @@ static ec_error_t op_move_same(const rulexec_in &rp,
 	rex.message_id = dst_mid;
 	if (exmdb_server::is_private() && rp.digest.has_value() &&
 	    common_util_get_mid_string(rp.sqlite, dst_mid, &pmid_string) &&
-	    pmid_string != nullptr) {
+	    pmid_string != nullptr)
 		(*rex.digest)["file"] = pmid_string;
-	}
+
 	auto ec = message_rule_new_message(std::move(rex), seen);
 	if (ec != ecSuccess)
 		return ec;
@@ -3786,26 +3786,21 @@ BOOL exmdb_server::deliver_message(const char *dir, const char *from_address,
 			return FALSE;
 	}
 
-	if (dlflags & DELIVERY_DO_NOTIF) {
-		auto dbase = pdb->lock_base_wr();
-		db_conn::NOTIFQ notifq;
-		for (const auto &mn : seen.msg) {
-			pdb->proc_dynamic_event(cpid, dynamic_event::new_msg,
-				mn.folder_id, mn.message_id, 0, *dbase, notifq);
-			if (message_id == mn.message_id)
-				pdb->notify_new_mail(mn.folder_id,
-					mn.message_id, *dbase, notifq);
-			else
-				pdb->notify_message_creation(mn.folder_id,
-					mn.message_id, *dbase, notifq);
-		}
-		if (sql_transact.commit() != SQLITE_OK)
-			return false;
-		dg_notify(std::move(notifq));
-	} else {
-		if (sql_transact.commit() != SQLITE_OK)
-			return false;
+	auto dbase = pdb->lock_base_wr();
+	db_conn::NOTIFQ notifq;
+	for (const auto &mn : seen.msg) {
+		pdb->proc_dynamic_event(cpid, dynamic_event::new_msg,
+			mn.folder_id, mn.message_id, 0, *dbase, notifq);
+		if (message_id == mn.message_id && dlflags & DELIVERY_DO_NOTIF)
+			pdb->notify_new_mail(mn.folder_id,
+				mn.message_id, *dbase, notifq);
+		else
+			pdb->notify_message_creation(mn.folder_id,
+				mn.message_id, *dbase, notifq);
 	}
+	if (sql_transact.commit() != SQLITE_OK)
+		return false;
+	dg_notify(std::move(notifq));
 	*new_folder_id = rop_util_make_eid_ex(1, fid_val);
 	*new_msg_id = rop_util_make_eid_ex(1, message_id);
 	*presult = static_cast<uint32_t>(partial ?

exch/exmdb/store2.cpp

@@ -64,7 +64,7 @@ BOOL exmdb_server::notify_new_mail(const char *dir, uint64_t folder_id,
 	auto pdb = db_engine_get_db(dir);
 	if (!pdb)
 		return false;
-	/* notify_new_mail needs an externally managed transaction on main. */
+	/* db_conn::notify_new_mail needs an externally managed transaction. So start one. */
 	auto sql_trans = gx_sql_begin(pdb->psqlite, txn_mode::read);
 	auto dbase = pdb->lock_base_wr();
 	db_conn::NOTIFQ notifq;

lib/ruleproc.cpp

@@ -126,6 +126,7 @@ struct rxparam {
 	ec_error_t is_oof(bool *out) const;
 	ec_error_t load_std_rules(bool oof, std::vector<rule_node> &out) const;
 	ec_error_t load_ext_rules(bool oof, std::vector<rule_node> &out) const;
+	void notify();
 
 	const char *ev_from = nullptr, *ev_to = nullptr;
 	message_node cur;
@@ -1325,11 +1326,30 @@ ec_error_t rxparam::run()
 			return err;
 	}
 
-	if (!exmdb_client->notify_new_mail(cur.dirc(), cur.fid, cur.mid))
-		mlog(LV_ERR, "ruleproc: newmail notification unsuccessful");
+	notify();
 	return ecSuccess;
 }
 
+void rxparam::notify()
+{
+	static constexpr proptag_t tagdata[] = {PR_MESSAGE_FLAGS, PR_MESSAGE_CLASS};
+	static constexpr PROPTAG_ARRAY tags = {std::size(tagdata), deconst(tagdata)};
+	TPROPVAL_ARRAY vals{};
+	if (!exmdb_client->get_message_properties(cur.dirc(), nullptr, CP_UTF8,
+	    cur.mid, &tags, &vals)) {
+		mlog(LV_ERR, "ruleproc: getprops FL/C failed");
+		return;
+	}
+	auto cls   = vals.get<const char>(PR_MESSAGE_CLASS);
+	auto flags = vals.get<const uint32_t>(PR_MESSAGE_FLAGS);
+	if (cls == nullptr || flags == nullptr) {
+		mlog(LV_ERR, "ruleproc: msg %llxh oddly missing FL/C", static_cast<unsigned long long>(cur.mid));
+		return;
+	}
+	if (!exmdb_client->transport_new_mail(cur.dirc(), cur.fid, cur.mid, *flags, cls))
+		mlog(LV_ERR, "ruleproc: tnewmail notification unsuccessful");
+}
+
 static ec_error_t exmdb_local_rules_execute(const char *dir, const char *ev_from,
     const char *ev_to, eid_t folder_id, eid_t msg_id, unsigned int flags) try
 {