midb: process deletions without delay

IMAP clients get really confused if a message is expunged but then reappears
as a supposedly new message, but carrying an old UID.

Suppose the state of the IMAP INBOX is:

```
a fetch 1:* (uid)
* 1 FETCH (UID 2359)
* 2 FETCH (UID 2360)
* 3 FETCH (UID 2361)
* 4 FETCH (UID 2362)
* 5 FETCH (UID 2363)
* 6 FETCH (UID 2364)
* 7 FETCH (UID 2365)
* 8 FETCH (UID 2366)
* 9 FETCH (UID 2367)
* 10 FETCH (UID 2368)
* 10 EXISTS
```

Deleting 1-5 may produce a response like so:

```
```
a store 1:5 +flags \deleted
* 1 FETCH (FLAGS (\Deleted))
* 2 FETCH (FLAGS (\Deleted))
* 3 FETCH (FLAGS (\Deleted))
* 4 FETCH (FLAGS (\Deleted))
* 5 FETCH (FLAGS (\Deleted))
a OK STORE completed
a expunge
* 5 EXPUNGE     #uid2363
* 4 EXPUNGE     #uid2362
* 3 EXPUNGE     #uid2361
* 2 EXPUNGE     #uid2360
* 1 EXPUNGE     #uid2359
* 8 EXISTS
* 0 RECENT
a OK EXPUNGE completed
```

After the EXPUNGE status lines, the IMAP client view is that INBOX
contains five messages (uids 2364-2368). ``8 EXISTS`` then indicates
that three new messages have come into existence. Per RFC 3501
§2.3.1.1, these must necessarily have higher UIDs >2368. This
condition is violated as a result of midb's delayed M-DELE processing
(the IMAP server's view is as follows):

```
a fetch 1:* (uid)
* 1 FETCH (UID 2361)
* 2 FETCH (UID 2362)
* 3 FETCH (UID 2363)
* 4 FETCH (UID 2364)
* 5 FETCH (UID 2365)
* 6 FETCH (UID 2366)
* 7 FETCH (UID 2367)
* 8 FETCH (UID 2368)
```

References: DESK-3220, DESK-3345

Author: jengelh

exch/midb/mail_engine.cpp

@@ -2233,7 +2233,6 @@ static int me_minst(int argc, char **argv, int sockd) try
  */
 static int me_mdele(int argc, char **argv, int sockd)
 {
-	int i;
 	BOOL b_partial;
 	EID_ARRAY message_ids;
 
@@ -2244,17 +2243,25 @@ static int me_mdele(int argc, char **argv, int sockd)
 	auto pidb = me_get_idb(argv[1]);
 	if (pidb == nullptr)
 		return MIDB_E_HASHTABLE_FULL;
+	auto pidb_transact = gx_sql_begin(pidb->psqlite, txn_mode::write);
+	if (!pidb_transact)
+		return false;
 	auto folder_id = me_get_folder_id(pidb.get(), argv[2]);
 	if (folder_id == 0)
 		return MIDB_E_NO_FOLDER;
 	unsigned int user_id = 0;
 	if (!mysql_adaptor_get_user_ids(pidb->username.c_str(), &user_id, nullptr, nullptr))
 		return MIDB_E_SSGETID;
+	/*
+	 * Modern SQLite (3.35) supports `DELETE FROM messages WHERE
+	 * mid_string=? RETURNING message_id,folder_id`, but RHEL9 only has
+	 * 3.34 still. So we need two separate queries.
+	 */
 	auto pstmt = gx_sql_prep(pidb->psqlite, "SELECT message_id,"
 	             " folder_id FROM messages WHERE mid_string=?");
 	if (pstmt == nullptr)
 		return MIDB_E_SQLPREP;
-	for (i=3; i<argc; i++) {
+	for (int i = 3; i < argc; ++i) {
 		sqlite3_reset(pstmt);
 		sqlite3_bind_text(pstmt, 1, argv[i], -1, SQLITE_STATIC);
 		if (SQLITE_ROW != pstmt.step() ||
@@ -2263,8 +2270,36 @@ static int me_mdele(int argc, char **argv, int sockd)
 		message_ids.pids[message_ids.count++] =
 			rop_util_make_eid_ex(1, sqlite3_column_int64(pstmt, 0));
 	}
+	/*
+	 * exmdb notifications may only arrive belatedly (or not at all),
+	 * therefore we should explicitly drop the messages from midb.sqlite
+	 * _now_, lest
+	 *
+	 * - the *same* midb client asking for a message listing
+	 *   after M-DELE could see messages again that really ought to be gone
+	 * - the *same* IMAP client asking for a message listing after EXPUNGE
+	 *   could see messages with IMAPUIDs that were just deleted, which can
+	 *   upset clients.
+	 *
+	 * While resynchronization can still "bring back" those messages (and
+	 * break midb client expectations), resync via RSYM/RSYF is considered
+	 * a debugging tool, and resync-upon-first-open of midb.sqlite is when
+	 * no clients are connected.
+	 */
+	pstmt = gx_sql_prep(pidb->psqlite, "DELETE FROM messages WHERE mid_string=?");
+	if (pstmt == nullptr)
+		return MIDB_E_SQLPREP;
+	for (int i = 3; i < argc; ++i) {
+		pstmt.reset();
+		pstmt.bind_text(1, argv[i]);
+		if (pstmt.step() != SQLITE_DONE)
+			/* ignore */;
+	}
 	pstmt.finalize();
+	if (pidb_transact.commit() != SQLITE_OK)
+		/* ignore */;
 	pidb.reset();
+
 	if (!exmdb_client->delete_messages(argv[1], CP_ACP, nullptr,
 	    rop_util_make_eid_ex(1, folder_id), &message_ids, TRUE, &b_partial))
 		return MIDB_E_MDB_DELETEMESSAGES;

mra/imap/parser.cpp

@@ -1281,6 +1281,11 @@ void imap_parser_echo_modify(imap_context *pcontext, STREAM *pstream)
 	hl_hold.unlock();
 
 	imap_parser_echo_expunges(*pcontext, pstream, f_expunged);
+	/*
+	 * 3501 §7: "the server checks the mailbox for new messages as part of
+	 * command execution. […] If new messages are found, the server sends
+	 * untagged EXISTS and RECENT responses."
+	 */
 	if (pcontext->contents.refresh(*pcontext, pcontext->selected_folder,
 	    f_expunged.size() > 0) == 0) {
 		auto outlen = gx_snprintf(buff, std::size(buff),