[WIP] executor: collect and return audit logs

Implemented the audit log extraction per program. This is available for
VMs that run only one proc and add `audit` in the experimental config.
The messages are appended to the output of the program and are printed
during result processing.

Author: winrares

executor/common_linux.h

@@ -4415,7 +4415,6 @@ inline int symlink(const char* old_path, const char* new_path)
 #define SYSTEM_UID 1000
 #define SYSTEM_GID 1000
 
-const char* const SELINUX_CONTEXT_UNTRUSTED_APP = "u:r:untrusted_app:s0:c512,c768";
 const char* const SELINUX_LABEL_APP_DATA_FILE = "u:object_r:app_data_file:s0:c512,c768";
 const char* const SELINUX_CONTEXT_FILE = "/proc/thread-self/attr/current";
 const char* const SELINUX_XATTR_NAME = "security.selinux";
@@ -4567,8 +4566,6 @@ static int do_sandbox_android(uint64 sandbox_arg)
 	prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
 
 	setfilecon(".", SELINUX_LABEL_APP_DATA_FILE);
-	if (uid == UNTRUSTED_APP_UID)
-		setcon(SELINUX_CONTEXT_UNTRUSTED_APP);
 
 	loop();
 	doexit(1);

executor/executor_runner.h

@@ -5,6 +5,7 @@
 #include <signal.h>
 #include <sys/mman.h>
 #include <sys/resource.h>
+#include <sys/socket.h>
 #include <unistd.h>
 
 #include <algorithm>
@@ -17,6 +18,21 @@
 #include <utility>
 #include <vector>
 
+#include <linux/audit.h>
+#include <linux/netlink.h>
+
+constexpr int NETLINK_BUF_SIZE = 4096;
+
+ssize_t ReceiveNetlinkMessage(int fd, void* buf, size_t len)
+{
+	return recv(fd, buf, len, 0);
+}
+
+ssize_t SendNetlinkMessage(int fd, const void* buf, size_t len)
+{
+	return send(fd, buf, len, 0);
+}
+
 inline std::ostream& operator<<(std::ostream& ss, const rpc::ExecRequestRawT& req)
 {
 	return ss << "id=" << req.id
@@ -109,7 +125,7 @@ class Proc
 {
 public:
 	Proc(Connection& conn, const char* bin, ProcIDPool& proc_id_pool, int& restarting, const bool& corpus_triaged, int max_signal_fd,
-	     int cover_filter_fd, ProcOpts opts)
+	     int cover_filter_fd, ProcOpts opts, int audit_sock)
 	    : conn_(conn),
 	      bin_(bin),
 	      proc_id_pool_(proc_id_pool),
@@ -121,7 +137,8 @@ class Proc
 	      opts_(opts),
 	      req_shmem_(kMaxInput),
 	      resp_shmem_(kMaxOutput),
-	      resp_mem_(static_cast<OutputData*>(resp_shmem_.Mem()))
+	      resp_mem_(static_cast<OutputData*>(resp_shmem_.Mem())),
+		  audit_sock_(audit_sock)
 	{
 		Start();
 	}
@@ -237,6 +254,7 @@ class Proc
 	uint64 exec_start_ = 0;
 	uint64 wait_start_ = 0;
 	uint64 wait_end_ = 0;
+	int audit_sock_ = 0;
 
 	friend std::ostream& operator<<(std::ostream& ss, const Proc& proc)
 	{
@@ -251,6 +269,63 @@ class Proc
 		return ss;
 	}
 
+	ssize_t SendUserAuditMessage(const std::string_view message_text) {
+		const size_t payload_len = message_text.length() + 1;
+		const size_t buf_len = NLMSG_SPACE(payload_len);
+		std::vector<char> buf(buf_len);
+		memset(buf.data(), 0, buf_len);
+
+		auto* nlh = reinterpret_cast<struct nlmsghdr*>(buf.data());
+		nlh->nlmsg_len = NLMSG_LENGTH((int)payload_len);
+		nlh->nlmsg_type = AUDIT_USER_AVC;
+		nlh->nlmsg_flags = NLM_F_REQUEST;
+
+		char* data = static_cast<char*>(NLMSG_DATA(nlh));
+		strncpy(data, message_text.data(), payload_len);
+		return SendNetlinkMessage(audit_sock_, nlh, nlh->nlmsg_len);
+	}
+
+	void drainAuditBacklog(std::vector<uint8_t>* output)
+	{
+		bool prefixed = false;
+		ssize_t slen = 0;
+		char buf[NETLINK_BUF_SIZE];
+		struct nlmsghdr* header;
+
+		if (SendUserAuditMessage("PROC END") < 0)
+			return;
+
+		// Drain the audit backlog until there is no other message
+		do {
+			slen = ReceiveNetlinkMessage(audit_sock_, buf, sizeof(buf));
+			if (errno == EINTR) {
+				continue;
+			}
+			if (slen < NLMSG_LENGTH(0)) {
+				fprintf(stderr, "audit: message too short\n");
+				continue;
+			}
+			header = (struct nlmsghdr*)buf;
+			if (header->nlmsg_type != AUDIT_AVC && header->nlmsg_type != AUDIT_USER_AVC) {
+				continue;
+			}
+			if (header->nlmsg_type == AUDIT_AVC) {
+				if (!prefixed) {
+					char tmp[128];
+					// Add prefix to the audit messages.
+					snprintf(tmp, sizeof(tmp), "\nAudit messages:\n");
+					output->insert(output->end(), tmp, tmp + strlen(tmp));
+					prefixed = true;
+				}
+				std::string message((char*)NLMSG_DATA(header),
+								(char*)NLMSG_DATA(header) +
+								(slen - sizeof(*header)));
+				message.append("\n");
+				output->insert(output->end(), message.c_str(), message.c_str() + strlen(message.c_str()));
+			}
+		} while (header->nlmsg_type != AUDIT_USER_AVC);
+	}
+
 	void ChangeState(State state)
 	{
 		if (state_ == State::Handshaking)
@@ -454,6 +529,10 @@ class Proc
 				output_.insert(output_.end(), tmp, tmp + strlen(tmp));
 			}
 		}
+		if (IsSet(msg_->flags, rpc::RequestFlag::ReturnAudit)) {
+			output = &output_;
+			drainAuditBacklog(output);
+		}
 		uint32 num_calls = 0;
 		if (msg_->type == rpc::RequestType::Program)
 			num_calls = read_input(&prog_data);
@@ -556,9 +635,13 @@ class Runner
 		proc_id_pool_.emplace(num_procs);
 		int max_signal_fd = max_signal_ ? max_signal_->FD() : -1;
 		int cover_filter_fd = cover_filter_ ? cover_filter_->FD() : -1;
+		int audit_sock = 0;
+		if (audit) {
+			audit_sock = registerForAudit();
+		}
 		for (int i = 0; i < num_procs; i++)
 			procs_.emplace_back(new Proc(conn, bin, *proc_id_pool_, restarting_, corpus_triaged_,
-						     max_signal_fd, cover_filter_fd, proc_opts_));
+						     max_signal_fd, cover_filter_fd, proc_opts_, audit_sock));
 
 		for (;;)
 			Loop();
@@ -574,6 +657,7 @@ class Runner
 	std::deque<rpc::ExecRequestRawT> requests_;
 	std::vector<std::string> leak_frames_;
 	int restarting_ = 0;
+	bool audit = false;
 	bool corpus_triaged_ = false;
 	ProcOpts proc_opts_{};
 
@@ -595,6 +679,71 @@ class Runner
 		return ss;
 	}
 
+	// Helper function to open a Netlink socket for Audit
+	int OpenNetlinkAuditSocket()
+	{
+		return socket(AF_NETLINK, SOCK_RAW, NETLINK_AUDIT);
+	}
+
+	int registerForAudit()
+	{
+		struct {
+			struct nlmsghdr nlh;
+			struct audit_status status;
+		} req;
+		memset(&req, 0, sizeof(req));
+
+		int fd = OpenNetlinkAuditSocket();
+		if (fd < 0) {
+			return -1;
+		}
+
+		req.nlh.nlmsg_len = NLMSG_LENGTH(sizeof(struct audit_status));
+		req.nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
+		req.nlh.nlmsg_seq = 1;
+		req.nlh.nlmsg_type = AUDIT_SET;
+		req.status.pid = getpid();
+		req.status.backlog_limit = 0;
+		req.status.rate_limit = 0;
+		req.status.backlog_wait_time = 0;
+		req.status.mask = AUDIT_STATUS_PID | AUDIT_STATUS_BACKLOG_LIMIT | AUDIT_STATUS_RATE_LIMIT | AUDIT_STATUS_BACKLOG_WAIT_TIME;
+
+		ssize_t sent = SendNetlinkMessage(fd, &req, req.nlh.nlmsg_len);
+		if (sent != req.nlh.nlmsg_len) {
+			close(fd);
+			return -1;
+		}
+
+		ssize_t slen = 0;
+		char buf[NETLINK_BUF_SIZE];
+		struct nlmsghdr* header;
+		do {
+			slen = ReceiveNetlinkMessage(fd, buf, sizeof(buf));
+			if (errno == EAGAIN || errno == EINTR) {
+				continue;
+			}
+			if (slen < NLMSG_LENGTH(0)) {
+				fprintf(stderr, "audit: message too short\n");
+				continue;
+			}
+			header = (struct nlmsghdr*)buf;
+		} while (header->nlmsg_type != NLMSG_ERROR); 
+
+		struct nlmsgerr* err;
+		if ((size_t)slen < NLMSG_LENGTH(sizeof(*err))) {
+			fprintf(stderr, "audit_listener: error message too short\n");
+			close(fd);
+			return -1;
+		}
+		err = (struct nlmsgerr*)NLMSG_DATA(header);
+		if (err->error != 0) {
+			fprintf(stderr, "audit_listener: received error %d\n", -err->error);
+			close(fd);
+			return -1;
+		}
+		return fd;
+	}
+
 	void Loop()
 	{
 		Select select;
@@ -661,6 +810,12 @@ class Runner
 		conn_.Recv(conn_reply);
 		if (conn_reply.debug)
 			flag_debug = true;
+		if (conn_reply.audit) {
+			if (conn_reply.procs > 1)
+				debug("audit only supported with one proc");
+			audit = conn_reply.procs == 1;
+		}
+
 		debug("connected to manager: procs=%d cover_edges=%d kernel_64_bit=%d slowdown=%d syscall_timeout=%u"
 		      " program_timeout=%u features=0x%llx\n",
 		      conn_reply.procs, conn_reply.cover_edges, conn_reply.kernel_64_bit,

pkg/flatrpc/flatrpc.fbs

@@ -50,6 +50,7 @@ table ConnectRequestRaw {
 
 table ConnectReplyRaw {
 	debug			:bool;
+	audit			:bool;
 	cover			:bool;
 	cover_edges		:bool;
 	kernel_64_bit		:bool;
@@ -128,6 +129,8 @@ enum RequestType : uint64 {
 enum RequestFlag : uint64 (bit_flags) {
 	// If set, collect program output and return in output field.
 	ReturnOutput,
+	// If set, collect audit logs produced by the program at the end of the output.
+	ReturnAudit,
 	// If set, don't fail on program failures, instead return the error in error field.
 	ReturnError,
 }