Return library reload failures to daemon control

Recoverable library errors in trust reload and RPM backend loader paths could terminate the daemon from inside src/library. Runtime backend reload failures also closed the live trust database, which risked losing the previous generation instead of leaving reload failure handling under daemon control.

The reload path now returns explicit errors, records trust reload failures, preserves or reopens the previous LMDB generation on reload sizing failures, and keeps the active database open when runtime backend init/load fails. The RPM loader parent now returns backend load failure for socket and loader IPC failures instead of calling exit(), while documented LRU aborts remain limited to cache corruption invariants.

Tests add fault coverage for libmagic initialization failure, trust reload failure preservation, and RPM loader IPC failure.

Author: stevegrubb

src/library/database.c

@@ -5516,12 +5516,12 @@ static void finish_trust_database_reload(void)
 	atomic_store_explicit(&reload_db_active, false, memory_order_release);
 }
 
-static void do_reload_db(conf_t* config)
+static int do_reload_db(conf_t *config)
 {
 	msg(LOG_INFO,
 	    "It looks like there was an update of the system... Syncing DB.");
 
-	int rc;
+	int rc = 0;
 	unsigned int old_db_max_size = config->db_max_size;
 	unsigned int old_reload_floor_mb = autosize_reload_floor_mb;
 
@@ -5534,30 +5534,54 @@ static void do_reload_db(conf_t* config)
 			config->db_max_size);
 
 		if (config->db_max_size < old_db_max_size) {
+			unsigned long old_trust_generation;
+			unsigned long old_env_generation;
+
 			/*
 			 * This is a map-size reopen, not controlled
 			 * compaction. If high-water usage needs physical file
 			 * rewrite, status reports recommend the explicit admin
 			 * compaction command instead of hiding a swap in reload.
 			 */
+			old_trust_generation =
+				trust_db_current_generation_number();
+			lmdb_environment_snapshot(&old_env_generation, NULL);
+			if (old_env_generation == 0)
+				old_env_generation = 1;
+
 			lock_update_thread();
-			close_env(0);
+			close_env(1);
+
+			rc = init_db_with_generations(config,
+				old_trust_generation, old_env_generation);
+			if (rc) {
+				int reopen_rc;
 
-			rc = init_db(config);
+				msg(LOG_ERR,
+				    "Cannot open the resized trust database, init_db() (%d)",
+				    rc);
+				config->db_max_size = old_db_max_size;
+				autosize_reload_floor_mb = old_reload_floor_mb;
+				reopen_rc = init_db_with_generations(config,
+					old_trust_generation,
+					old_env_generation);
+				if (reopen_rc) {
+					msg(LOG_ERR,
+					    "Cannot reopen previous trust database, init_db() (%d)",
+					    reopen_rc);
+					rc = reopen_rc;
+				} else
+					msg(LOG_ERR,
+					    "Previous trust database preserved after reload sizing failure");
+			}
 			unlock_update_thread();
 
 			if (rc) {
-				msg(LOG_ERR,
-			     "Cannot open the trust database, init_db() (%d)",
-					rc);
 				if (stop)
 					goto out;
 
 				record_trust_reload_failure();
-				close(ffd[0].fd);
-				backend_close();
-				unlink_fifo();
-				exit(rc);
+				goto out;
 			}
 		} else if (config->db_max_size > old_db_max_size) {
 			lock_update_thread();
@@ -5579,14 +5603,12 @@ static void do_reload_db(conf_t* config)
 	if ((rc = backend_init(config))) {
 		msg(LOG_ERR, "Failed to load trust data from backend (%d)", rc);
 		record_trust_reload_failure();
-		close_db(0);
 		goto out;
 	}
 
 	if ((rc = backend_load(config))) {
 		msg(LOG_ERR, "Failed to load data from backend (%d)", rc);
 		record_trust_reload_failure();
-		close_db(0);
 		goto out;
 	}
 
@@ -5598,24 +5620,33 @@ static void do_reload_db(conf_t* config)
 
 		record_trust_reload_failure();
 		if (rc == UPDATE_DB_PRESERVED) {
-			// update_database() failed before clearing the live DB.
-			// Keep running with the last successfully built trust set.
 			msg(LOG_ERR,
 			    "Previous trust database preserved after reload failure");
 			goto out;
 		}
 
-		close(ffd[0].fd);
-		backend_close();
-		unlink_fifo();
-		exit(rc);
+		msg(LOG_ERR,
+		    "Trust database reload failed with active DB under daemon control");
+		goto out;
 	}
 
 	msg(LOG_INFO, "Updated");
 
 out:
 	// Conserve memory
 	backend_close();
+	return rc;
+}
+
+/*
+ * database_reload_for_tests - run the trust reload path directly.
+ * @config: test configuration containing backend and LMDB settings.
+ *
+ * Returns 0 on success and non-zero when reload failed.
+ */
+int database_reload_for_tests(conf_t *config)
+{
+	return do_reload_db(config);
 }
 
 /*

src/library/database.h

@@ -92,6 +92,7 @@ int database_publish_memfd_for_tests(int memfd, conf_t *config)
 int database_publish_startup_memfd_for_tests(int memfd, conf_t *config)
 	__nonnull ((2));
 int database_drop_candidate_after_import_for_tests(int memfd);
+int database_reload_for_tests(conf_t *config) __nonnull ((1));
 int database_compact_memfd_for_tests(int memfd, conf_t *config)
 	__nonnull ((2));
 void *database_generation_hold_for_tests(void);

src/library/fapolicyd-backend.h

@@ -51,4 +51,9 @@ typedef struct _backend
 	long entries;
 } backend;
 
+#ifdef USE_RPM
+int rpm_backend_load_from_path_for_tests(const conf_t *conf,
+	const char *loader_path) __nonnull ((1, 2));
+#endif
+
 #endif

src/library/file.c

@@ -60,6 +60,7 @@ static char *get_program_cwd_from_pid(pid_t pid, size_t blen, char *buf)
 				__attr_access ((__write_only__, 3, 2));
 static void resolve_path(const char *pcwd, char *path, size_t len)
 				__attr_access ((__write_only__, 2, 3));
+static file_init_status_t file_init_failure;
 
 /*
  * file_close_context - release file helper state owned by a decision context.
@@ -123,6 +124,12 @@ int file_init(void)
 
 	// Setup libmagic
 	unsetenv("MAGIC");
+	if (file_init_failure == FILE_INIT_MAGIC_FAST_OPEN_FAILED) {
+		msg(LOG_ERR, "Unable to init fast libmagic");
+		file_close_context(ctx);
+		return FILE_INIT_MAGIC_FAST_OPEN_FAILED;
+	}
+
 	// Fast magic: minimal rules, all expensive checks disabled
 	ctx->magic_fast = magic_open(
 		MAGIC_MIME |
@@ -136,33 +143,48 @@ int file_init(void)
 		MAGIC_NO_CHECK_JSON        /* Skip JSON validation */
 		);
 	if (ctx->magic_fast == NULL) {
-		msg(LOG_ERR, "Unable to init libmagic");
+		msg(LOG_ERR, "Unable to init fast libmagic");
 		file_close_context(ctx);
-		return 1;
+		return FILE_INIT_MAGIC_FAST_OPEN_FAILED;
 	}
 
 	// Load only essential magic rules
+	if (file_init_failure == FILE_INIT_MAGIC_FAST_LOAD_FAILED) {
+		msg(LOG_ERR, "Unable to load fast magic database");
+		file_close_context(ctx);
+		return FILE_INIT_MAGIC_FAST_LOAD_FAILED;
+	}
 	if (magic_load(ctx->magic_fast, MAGIC_PATH) != 0) {
 		msg(LOG_ERR, "Unable to load fast magic database");
 		file_close_context(ctx);
-		return 2;
+		return FILE_INIT_MAGIC_FAST_LOAD_FAILED;
 	}
 
 	// Full magic: normal operation
+	if (file_init_failure == FILE_INIT_MAGIC_FULL_OPEN_FAILED) {
+		msg(LOG_ERR, "Unable to init full libmagic");
+		file_close_context(ctx);
+		return FILE_INIT_MAGIC_FULL_OPEN_FAILED;
+	}
 	ctx->magic_full = magic_open(MAGIC_MIME|MAGIC_ERROR|MAGIC_NO_CHECK_CDF|
 			MAGIC_NO_CHECK_ELF);
 	if (ctx->magic_full == NULL) {
-		msg(LOG_ERR, "Unable to init libmagic");
+		msg(LOG_ERR, "Unable to init full libmagic");
 		file_close_context(ctx);
-		return 3;
+		return FILE_INIT_MAGIC_FULL_OPEN_FAILED;
 	}
 	// System default
+	if (file_init_failure == FILE_INIT_MAGIC_FULL_LOAD_FAILED) {
+		msg(LOG_ERR, "Unable to load default magic database");
+		file_close_context(ctx);
+		return FILE_INIT_MAGIC_FULL_LOAD_FAILED;
+	}
 	if (magic_load(ctx->magic_full, NULL) != 0) {
 		msg(LOG_ERR, "Unable to load default magic database");
 		file_close_context(ctx);
-		return 4;
+		return FILE_INIT_MAGIC_FULL_LOAD_FAILED;
 	}
-	return 0;
+	return FILE_INIT_OK;
 }
 
 
@@ -173,6 +195,18 @@ void file_close(void)
 }
 
 
+/*
+ * file_init_failure_for_tests - request one file_init() fault injection.
+ * @failure: file_init_status_t value to return, or FILE_INIT_OK to disable.
+ *
+ * Returns nothing.
+ */
+void file_init_failure_for_tests(file_init_status_t failure)
+{
+	file_init_failure = failure;
+}
+
+
 /*
  * file_hash_length - return the binary digest size for the algorithm.
  * @alg: file digest algorithm to query.

src/library/file.h

@@ -39,6 +39,14 @@ typedef enum {
 	FILE_HASH_ALG_SHA512,
 } file_hash_alg_t;
 
+typedef enum {
+	FILE_INIT_OK = 0,
+	FILE_INIT_MAGIC_FAST_OPEN_FAILED = 1,
+	FILE_INIT_MAGIC_FAST_LOAD_FAILED = 2,
+	FILE_INIT_MAGIC_FULL_OPEN_FAILED = 3,
+	FILE_INIT_MAGIC_FULL_LOAD_FAILED = 4,
+} file_init_status_t;
+
 #define MD5_LEN		16
 #define SHA1_LEN	20
 #define SHA256_LEN	32
@@ -63,6 +71,7 @@ struct file_info
 };
 
 int file_init(void);
+void file_init_failure_for_tests(file_init_status_t failure);
 void file_close(void);
 struct file_info *stat_file_entry(int fd) __attr_dealloc_free;
 void file_info_reset_digest(struct file_info *info);

src/library/lru.c

@@ -270,6 +270,11 @@ static void remove_node(Queue *queue, const QNode *node)
 		goto out;
 	} else {
 		if (node->prev->next != node) {
+			/*
+			 * The LRU cache owns enforcement objects. Once neighbor
+			 * links disagree, continuing can detach or free the
+			 * wrong object, so this remains an invariant failure.
+			 */
 			msg(LOG_ERR, "Linked list corruption detected %s",
 				queue->name);
 			abort();
@@ -284,6 +289,11 @@ static void remove_node(Queue *queue, const QNode *node)
 			queue->end->next = NULL;
 	} else {
 		if (node->next->prev != node) {
+			/*
+			 * The LRU cache owns enforcement objects. Once neighbor
+			 * links disagree, continuing can detach or free the
+			 * wrong object, so this remains an invariant failure.
+			 */
 			msg(LOG_ERR, "Linked List corruption detected %s",
 				queue->name);
 			abort();
@@ -339,6 +349,11 @@ void lru_evict(Queue *queue, unsigned int key)
 	QNode *temp = queue->front;
 
 	if (hash->array[key] != temp) {
+		/*
+		 * Key-to-node mismatch means the hash table and eviction list
+		 * no longer describe the same cache. Evicting after that could
+		 * corrupt enforcement state, so stop at the invariant failure.
+		 */
 		msg(LOG_ERR, "lru_evict called with mismatched key %s",
 			queue->name);
 		abort();

src/library/rpm-backend.c

@@ -217,13 +217,15 @@ struct _hash_record {
 
 #define BUFFER_SIZE 4096
 #define MAX_DELIMS 3	// Trustdb has 4 fields - therefore 3 delimiters
+static const char *rpm_loader_path = BINARYDIR "/fapolicyd-rpm-loader";
+
 static int rpm_load_list(const conf_t *conf)
 {
 	// before the spawn
 	int sv[2];
 	if (socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sv) < 0) {
 		msg(LOG_ERR, "socketpair failed");
-		exit(1);
+		return 1;
 	}
 
 	posix_spawn_file_actions_t actions;
@@ -238,7 +240,7 @@ static int rpm_load_list(const conf_t *conf)
 	char *custom_env[] = { "FAPO_SOCK_FD=3", NULL };
 
 	pid_t pid = -1;
-	int status = posix_spawn(&pid, BINARYDIR "/fapolicyd-rpm-loader",
+	int status = posix_spawn(&pid, rpm_loader_path,
 					 &actions, NULL, argv, custom_env);
 	close(sv[1]);  // Parent doesn't write
 
@@ -258,15 +260,20 @@ static int rpm_load_list(const conf_t *conf)
 		_msg.msg_controllen = sizeof cmsgbuf.buf;
 
 		if (recvmsg(sv[0], &_msg, 0) < 0) {
-			msg(LOG_ERR, "recvmesg failed");
-			exit(1);
+			msg(LOG_ERR, "recvmsg failed");
+			close(sv[0]);
+			waitpid(pid, &status, 0);
+			posix_spawn_file_actions_destroy(&actions);
+			return 1;
 		}
 		close(sv[0]);
 
 		struct cmsghdr *c = CMSG_FIRSTHDR(&_msg);
 		if (!c || c->cmsg_type != SCM_RIGHTS) {
 			msg(LOG_ERR, "missing fd");
-			exit(1);
+			waitpid(pid, &status, 0);
+			posix_spawn_file_actions_destroy(&actions);
+			return 1;
 		}
 
 		int memfd;
@@ -286,8 +293,10 @@ static int rpm_load_list(const conf_t *conf)
 		rpm_backend.memfd = memfd;
 
 		waitpid(pid, &status, 0);
-	} else
+	} else {
+		close(sv[0]);
 		msg(LOG_ERR, "posix_spawn failed: %s\n", strerror(status));
+	}
 
 	posix_spawn_file_actions_destroy(&actions);
 
@@ -305,6 +314,25 @@ static int rpm_load_list(const conf_t *conf)
 	return exit_rc;
 }
 
+/*
+ * rpm_backend_load_from_path_for_tests - exercise rpm loader IPC errors.
+ * @conf: test configuration.
+ * @loader_path: helper executable path to spawn.
+ *
+ * Returns the rpm backend load result without terminating the caller.
+ */
+int rpm_backend_load_from_path_for_tests(const conf_t *conf,
+					 const char *loader_path)
+{
+	const char *previous_loader_path = rpm_loader_path;
+	int rc;
+
+	rpm_loader_path = loader_path;
+	rc = rpm_load_list(conf);
+	rpm_loader_path = previous_loader_path;
+	return rc;
+}
+
 // this function is used in fapolicyd-rpm-loader
 extern unsigned int debug_mode;
 int do_rpm_load_list(const conf_t *conf, int memfd)