Merge branch 'prime'

Author: jengelh

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([gromox], [2.42])
+AC_INIT([gromox], [2.43])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_MACRO_DIR([build-aux])
 AC_PREFIX_DEFAULT([/usr])

doc/changelog.rst

@@ -1,3 +1,29 @@
+Gromox 2.43 (2025-03-06)
+========================
+
+Fixes:
+
+* imap: reduce memory footprint when FETCHing messages
+* imap: resolve shutdown ordering crash
+* exmdb: upgrade 0-length PR_ENTRYIDs to fake 1 byte to soothe Outlook Cached
+  Mode syncer
+* ews: heed MAPI proptype semantics and interpret 0xffffffff as a signed int
+  when serializing to XML (-1 rather than 4294967295)
+* gab: resolve out-of-bounds access in dntomid
+
+Enhancements:
+
+* oxm2mt: support embedded messages
+* oxm2mt: add option to extract just an attachment embedded message
+* eml2mt: add option to extract just an attachment embedded message
+* mysql_adaptor: add TLS connection config directives
+
+Behavioral changes:
+
+* exmdb: launch threads in more rapid succession when there is job queue
+  contention
+
+
 Gromox 2.42 (2025-02-19)
 ========================
 

doc/exchange_nsp.4gx

@@ -19,7 +19,8 @@ Default: \fI3000\fP
 .TP
 \fBnsp_trace\fP
 Level 1: Log entry into and exit out of NSP functions, with their parameter
-values. Log data dumps of select calls. Level 2: Dump more data.
+values to stderr (not http_log_file!). Log data dumps of select calls. Level 2:
+Dump more data.
 .br
 Default: \fI0\fP
 .TP

doc/midb.8gx

@@ -67,8 +67,9 @@ Default: \fI30minutes\fP
 .TP
 \fBmidb_cmd_debug\fP
 Log every incoming MIDB command and the return code of the operation in a
-minimal fashion to stderr. Level 1 emits commands with a failure return code,
-level 2 emits all commands.
+minimal fashion to stderr (not midb_log_file!). Level 1 emits commands with a
+failure return code, level 2 emits all commands. Logs are written to stderr
+only.
 .br
 Default: \fI0\fP
 .TP

doc/pop3.8gx

@@ -117,8 +117,8 @@ Default: (unset)
 .TP
 \fBpop3_cmd_debug\fP
 Log every incoming POP3 command and the return code of the operation in a
-minimal fashion to stderr. Level 1 emits commands that have failed execution,
-level 2 emits all commands.
+minimal fashion to stderr (not pop3_log_file!). Level 1 emits commands that
+have failed execution, level 2 emits all commands.
 .br
 Default: \fI0\fP
 .TP

exch/http/pdu_processor.cpp

@@ -1522,15 +1522,15 @@ static uint32_t pdu_processor_apply_async_id()
 	if (g_async_hash.size() >= 2 * ctx_num) {
 		as_hold.unlock();
 		delete pasync_node;
-		fprintf(stderr, "E-2045: g_async_hash reached maximum fill level (influenced by http.cfg:context_num,connection_ratio)\n");
+		mlog(LV_ERR, "E-2045: g_async_hash reached maximum fill level (influenced by http.cfg:context_num,connection_ratio)\n");
 		return 0;
 	}
 	try {
 		g_async_hash.emplace(async_id, nullptr);
 	} catch (const std::bad_alloc &) {
 		as_hold.unlock();
 		delete pasync_node;
-		fprintf(stderr, "E-2044: ENOMEM\n");
+		mlog(LV_ERR, "E-2044: ENOMEM\n");
 		return 0;
 	}
 	pasync_node->async_id = async_id;

exch/oab.cpp

@@ -67,7 +67,7 @@ http_status OabPlugin::proc(int ctx_id, const void *content, uint64_t len) try
 		return wr;
 	return write_response(ctx_id, response, strlen(response));
 } catch (const std::bad_alloc &) {
-	fprintf(stderr, "E-2092: ENOMEM\n");
+	mlog(LV_ERR, "E-2092: ENOMEM\n");
 	return http_status::none;
 }
 

exch/oxdisco.cpp

@@ -355,7 +355,7 @@ http_status OxdiscoPlugin::proc(int ctx_id, const void *content, uint64_t len) t
 		mlog(LV_DEBUG, "[oxdisco] send redirect response");
 	return resp(ctx_id, auth_info.username, email, ars);
 } catch (const std::bad_alloc &) {
-	fprintf(stderr, "E-1700: ENOMEM\n");
+	mlog(LV_ERR, "E-1700: ENOMEM\n");
 	return die(ctx_id, server_error_code, server_error_msg);
 }
 

include/gromox/mjson.hpp

@@ -15,12 +15,13 @@ struct GX_EXPORT mjson_io {
 	using c_iter = decltype(m_cache)::const_iterator;
 
 	bool exists(const std::string &path) const;
-	c_iter find(const std::string &path);
+	const std::string *get_full(const std::string &path) const;
+	std::optional<std::string> get_substr(const std::string &path, size_t of, size_t ln) const;
+	ssize_t get_size(const std::string &path) const;
 	void place(const std::string &path, std::string &&ctnt);
 	void clear() { m_cache.clear(); }
 	bool valid(c_iter it) const { return it != m_cache.cend(); }
 	bool invalid(c_iter it) const { return it == m_cache.cend(); }
-	static std::string substr(c_iter it, size_t of, size_t ln);
 };
 
 struct GX_EXPORT MJSON_MIME {

lib/email/mjson.cpp

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: 2020–2025 grommunio GmbH
 // This file is part of Gromox.
 #include <cerrno>
+#include <climits>
 #include <cstdio>
 #include <cstring>
 #include <fcntl.h>
@@ -63,29 +64,35 @@ bool mjson_io::exists(const std::string &path) const
 	return m_cache.find(path) != m_cache.cend();
 }
 
-mjson_io::c_iter mjson_io::find(const std::string &path)
+void mjson_io::place(const std::string &path, std::string &&content)
+{
+	m_cache[path] = std::move(content);
+}
+
+const std::string *mjson_io::get_full(const std::string &path) const
 {
 	auto iter = m_cache.find(path);
-	if (iter != m_cache.cend())
-		return iter;
-	/*
-	 * Not finding the element is not an error; asking for `BODY[2.1.3]`
-	 * will iteratively look for /2.1.3.dgt, /2.1.dgt, /2.dgt.
-	 */
-	return m_cache.cend();
+	return iter != m_cache.end() ? &iter->second : nullptr;
 }
 
-void mjson_io::place(const std::string &path, std::string &&content)
+ssize_t mjson_io::get_size(const std::string &path) const
 {
-	m_cache[path] = std::move(content);
+	auto str = get_full(path);
+	if (str == nullptr)
+		return -1;
+	if (str->size() >= SSIZE_MAX)
+		return SSIZE_MAX;
+	return str->size();
 }
 
-std::string mjson_io::substr(mjson_io::c_iter it, size_t of, size_t len)
+std::optional<std::string> mjson_io::get_substr(const std::string &path,
+    size_t of, size_t len) const
 {
-	const auto &str = it->second;
-	if (of <= str.size())
-		return str.substr(of, len);
-	return {};
+	std::optional<std::string> r;
+	auto str = get_full(path);
+	if (str == nullptr || of > str->size())
+		return r;
+	return r.emplace(str->substr(of, len));
 }
 
 bool MJSON_MIME::contains_none_type() const
@@ -364,10 +371,8 @@ static int mjson_fetch_mime_structure(mjson_io &io, const MJSON_MIME *pmime,
 				temp_path = storage_path + "/"s + pmime->get_id();
 			else
 				temp_path = storage_path + "/"s + msg_filename + "." + pmime->get_id();
-			auto fd = io.find(temp_path);
-			buf += io.valid(fd) ?
-			       " " + std::to_string(fd->second.size()) :
-			       " NIL";
+			ssize_t z = io.get_size(temp_path);
+			buf += z >= 0 ? " " + std::to_string(z) : " NIL";
 		} else {
 			buf += " " + std::to_string(pmime->length);
 		}
@@ -386,11 +391,11 @@ static int mjson_fetch_mime_structure(mjson_io &io, const MJSON_MIME *pmime,
 				temp_path = storage_path + "/"s + msg_filename +
 				            "." + pmime->get_id() + ".dgt";
 
-			auto fd = io.find(temp_path);
-			if (io.invalid(fd))
+			auto eml_content = io.get_full(temp_path);
+			if (eml_content == nullptr)
 				goto RFC822_FAILURE;
 			Json::Value digest;
-			if (!json_from_str(fd->second, digest))
+			if (!json_from_str(*eml_content, digest))
 				goto RFC822_FAILURE;
 			MJSON temp_mjson;
 			if (!temp_mjson.load_from_json(digest))
@@ -559,15 +564,15 @@ static void mjson_enum_build(const MJSON_MIME *pmime, BUILD_PARAM *pbuild) { try
 		dgt_path = msg_path + ".dgt";
 	}
 
-	auto iter = pbuild->io.find(temp_path);
-	if (pbuild->io.invalid(iter)) {
+	auto eml_content = pbuild->io.get_substr(temp_path, pmime->get_content_offset(),
+	                   pmime->get_content_length());
+	if (!eml_content.has_value()) {
 		pbuild->build_result = FALSE;
 		return;
 	}
-	
-	auto eml = mjson_io::substr(iter, pmime->get_content_offset(), pmime->get_content_length());
+	std::string eml;
 	if (pmime->encoding_is_b()) {
-		eml = base64_decode(eml);
+		eml = base64_decode(*std::move(eml_content));
 	} else if (pmime->encoding_is_q()) {
 		std::string qpout;
 		qpout.resize(eml.size());
@@ -665,12 +670,12 @@ BOOL MJSON::rfc822_get(mjson_io &io, MJSON *pjson, const char *storage_path,
 		char dgt_path[256];
 		snprintf(dgt_path, std::size(dgt_path), "%s/%s/%s.dgt", storage_path,
 		         pjson_base->get_mail_filename(), mjson_id);
-		auto fd = io.find(dgt_path);
-		if (io.invalid(fd))
+		auto eml_content = io.get_full(dgt_path);
+		if (eml_content == nullptr)
 			continue;
 		pjson->clear();
 		Json::Value digest;
-		if (!json_from_str(fd->second, digest) ||
+		if (!json_from_str(*eml_content, digest) ||
 		    !pjson->load_from_json(digest))
 			return false;
 		pjson->path = temp_path;