executor, sys/linux, pkg: enable syzos for riscv64

This patch enables syzos for riscv64 and implements
the corresponding pseudo syscalls.

Pseudo syscalls:
  - syz_kvm_setup_syzos_vm
  - syz_kvm_add_vcpu
  - syz_kvm_assert_syzos_uexit

Syzos guest support:
  - guest_uexit
  - guest_execute_code
  - guest_handle_csrr and guest_handle_csrw

Test seeds:
  - riscv64-syz_kvm_setup_syzos_vm
  - riscv64-syz_kvm_setup_syzos_vm-csrr
  - riscv64-syz_kvm_setup_syzos_vm-csrw

Author: 6eanut

executor/common_kvm_riscv64.h

@@ -8,17 +8,17 @@
 
 // Implementation of syz_kvm_setup_cpu pseudo-syscall.
 
+#include "common_kvm.h"
+#include "kvm.h"
 #include <stdint.h>
 #include <string.h>
 #include <sys/ioctl.h>
 
-#if SYZ_EXECUTOR || __NR_syz_kvm_setup_cpu
-struct kvm_text {
-	uintptr_t type;
-	const void* text;
-	uintptr_t size;
-};
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_syzos_vm || __NR_syz_kvm_add_vcpu
+#include "common_kvm_riscv64_syzos.h"
+#endif
 
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_cpu || __NR_syz_kvm_add_vcpu
 // Construct RISC-V register id for KVM.
 #define RISCV_CORE_REG(idx) (KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_CORE | (idx))
 #define RISCV_CSR_REG(idx) (KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_CSR | (idx))
@@ -83,9 +83,6 @@ enum riscv_core_index {
 // Indicate the Supervisor Interrupt Enable state.
 #define SSTATUS_SIE (1UL << 1)
 
-// Define the starting physical address for the guest code.
-#define CODE_START 0x80000000ULL
-
 // Set a single register value for the specified CPU file descriptor.
 static inline int kvm_set_reg(int cpufd, unsigned long id, unsigned long value)
 {
@@ -96,6 +93,59 @@ static inline int kvm_set_reg(int cpufd, unsigned long id, unsigned long value)
 	return ioctl(cpufd, KVM_SET_ONE_REG, &reg);
 }
 
+struct kvm_text {
+	uintptr_t type;
+	const void* text;
+	uintptr_t size;
+};
+#endif
+
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_cpu || __NR_syz_kvm_setup_syzos_vm || __NR_syz_kvm_add_vcpu
+#define KVM_RISCV_SELFTESTS_SBI_EXT 0x08FFFFFF
+#define KVM_RISCV_SELFTESTS_SBI_UNEXP 1
+
+struct sbiret {
+	long error;
+	long value;
+};
+
+struct sbiret sbi_ecall(unsigned long arg0, unsigned long arg1,
+			unsigned long arg2, unsigned long arg3,
+			unsigned long arg4, unsigned long arg5,
+			int fid, int ext)
+{
+	struct sbiret ret;
+
+	register uintptr_t a0 asm("a0") = (uintptr_t)(arg0);
+	register uintptr_t a1 asm("a1") = (uintptr_t)(arg1);
+	register uintptr_t a2 asm("a2") = (uintptr_t)(arg2);
+	register uintptr_t a3 asm("a3") = (uintptr_t)(arg3);
+	register uintptr_t a4 asm("a4") = (uintptr_t)(arg4);
+	register uintptr_t a5 asm("a5") = (uintptr_t)(arg5);
+	register uintptr_t a6 asm("a6") = (uintptr_t)(fid);
+	register uintptr_t a7 asm("a7") = (uintptr_t)(ext);
+	asm volatile("ecall"
+		     : "+r"(a0), "+r"(a1)
+		     : "r"(a2), "r"(a3), "r"(a4), "r"(a5), "r"(a6), "r"(a7)
+		     : "memory");
+	ret.error = a0;
+	ret.value = a1;
+
+	return ret;
+}
+
+__attribute__((used)) __attribute((__aligned__(16))) static void guest_unexp_trap(void)
+{
+	sbi_ecall(0, 0, 0, 0, 0, 0,
+		  KVM_RISCV_SELFTESTS_SBI_UNEXP,
+		  KVM_RISCV_SELFTESTS_SBI_EXT);
+}
+#endif
+
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_cpu
+// Define the starting physical address for the guest code.
+#define CODE_START 0x80000000ULL
+
 // syz_kvm_setup_cpu$riscv64(fd fd_kvmvm, cpufd fd_kvmcpu, usermem vma[24], text ptr[in, array[kvm_text_riscv64, 1]], ntext len[text], flags const[0], opts ptr[in, array[kvm_setup_opt_riscv64, 1]], nopt len[opts])
 static volatile long syz_kvm_setup_cpu(volatile long a0, volatile long a1, volatile long a2, volatile long a3, volatile long a4, volatile long a5, volatile long a6, volatile long a7)
 {
@@ -131,6 +181,7 @@ static volatile long syz_kvm_setup_cpu(volatile long a0, volatile long a1, volat
 	if (size > guest_mem_size)
 		size = guest_mem_size;
 	memcpy(host_mem, text, size);
+	memcpy(host_mem + page_size, (void*)guest_unexp_trap, KVM_PAGE_SIZE);
 
 	// Initialize VCPU registers.
 	// Set PC (program counter) to start of code.
@@ -151,6 +202,14 @@ static volatile long syz_kvm_setup_cpu(volatile long a0, volatile long a1, volat
 	unsigned long stvec = CODE_START + page_size;
 	if (kvm_set_reg(cpufd, RISCV_CSR_REG(CSR_STVEC), stvec))
 		return -1;
+	// Set GP.
+	unsigned long current_gp = 0;
+	asm volatile("add %0, gp, zero"
+		     : "=r"(current_gp)
+		     :
+		     : "memory");
+	if (kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_GP), current_gp))
+		return -1;
 
 	return 0;
 }
@@ -175,4 +234,226 @@ static long syz_kvm_assert_reg(volatile long a0, volatile long a1, volatile long
 }
 #endif
 
-#endif // EXECUTOR_COMMON_KVM_RISCV64_H
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_syzos_vm || __NR_syz_kvm_add_vcpu
+struct kvm_syz_vm {
+	int vmfd;
+	int next_cpu_id;
+	void* user_text;
+};
+#endif
+
+#if SYZ_EXECUTOR || __NR_syz_kvm_setup_syzos_vm
+struct addr_size {
+	void* addr;
+	size_t size;
+};
+
+static struct addr_size alloc_guest_mem(struct addr_size* free, size_t size)
+{
+	struct addr_size ret = {.addr = NULL, .size = 0};
+
+	if (free->size < size)
+		return ret;
+	ret.addr = free->addr;
+	ret.size = size;
+	free->addr = (void*)((char*)free->addr + size);
+	free->size -= size;
+	return ret;
+}
+
+// Call KVM_SET_USER_MEMORY_REGION for the given pages.
+static void vm_set_user_memory_region(int vmfd, uint32 slot, uint32 flags, uint64 guest_phys_addr, uint64 memory_size, uint64 userspace_addr)
+{
+	struct kvm_userspace_memory_region memreg;
+	memreg.slot = slot;
+	memreg.flags = flags;
+	memreg.guest_phys_addr = guest_phys_addr;
+	memreg.memory_size = memory_size;
+	memreg.userspace_addr = userspace_addr;
+	ioctl(vmfd, KVM_SET_USER_MEMORY_REGION, &memreg);
+}
+
+#define AUIPC_OPCODE 0x17
+#define AUIPC_OPCODE_MASK 0x7f
+
+// Code loading SyzOS into guest memory does not handle data relocations (see
+// https://github.com/google/syzkaller/issues/5565), so SyzOS will crash soon after encountering an
+// AUIPC instruction. Detect these instructions to catch regressions early.
+// The most common reason for using data relocaions is accessing global variables and constants.
+// Sometimes the compiler may choose to emit a read-only constant to zero-initialize a structure
+// or to generate a jump table for a switch statement.
+static void validate_guest_code(void* mem, size_t size)
+{
+	uint32* insns = (uint32*)mem;
+	for (size_t i = 0; i < size / 4; i++) {
+		if ((insns[i] & AUIPC_OPCODE_MASK) == AUIPC_OPCODE)
+			fail("AUIPC instruction detected in SyzOS, exiting");
+	}
+}
+
+static void install_syzos_code(void* host_mem, size_t mem_size)
+{
+	size_t size = (char*)&__stop_guest - (char*)&__start_guest;
+	if (size > mem_size)
+		fail("SyzOS size exceeds guest memory");
+	memcpy(host_mem, &__start_guest, size);
+	validate_guest_code(host_mem, size);
+}
+
+static void setup_vm(int vmfd, void* host_mem, void** text_slot)
+{
+	// Guest physical memory layout (must be in sync with executor/kvm.h):
+	// 0x00000000 - unused pages
+	// 0x00001000 - exception vector table
+	// 0x02000000 - CLINT (MMIO, no memory allocated)
+	// 0x0c000000 - PLIC  (MMIO, no memory allocated)
+	// 0x40000000 - unmapped region to trigger a page faults for uexits etc. (1 page)
+	// 0x40001000 - writable region with KVM_MEM_LOG_DIRTY_PAGES to fuzz dirty ring (2 pages)
+	// 0x80000000 - user code (4 pages)
+	// 0x80008000 - executor guest code (4 pages)
+	// 0x80010000 - scratch memory for code generated at runtime (1 page)
+	// 0x80020000 - per-vCPU stacks (1 page each)
+	struct addr_size allocator = {.addr = host_mem, .size = KVM_GUEST_MEM_SIZE};
+	int slot = 0; // Slot numbers do not matter, they just have to be different.
+	// Setup executor guest code.
+	struct addr_size host_text = alloc_guest_mem(&allocator, 4 * KVM_PAGE_SIZE);
+	install_syzos_code(host_text.addr, host_text.size);
+	vm_set_user_memory_region(vmfd, slot++, KVM_MEM_READONLY, SYZOS_ADDR_EXECUTOR_CODE, host_text.size, (uintptr_t)host_text.addr);
+	// Setup writable region with KVM_MEM_LOG_DIRTY_PAGES to fuzz dirty ring.
+	struct addr_size next = alloc_guest_mem(&allocator, 2 * KVM_PAGE_SIZE);
+	vm_set_user_memory_region(vmfd, slot++, KVM_MEM_LOG_DIRTY_PAGES, RISCV64_ADDR_DIRTY_PAGES, next.size, (uintptr_t)next.addr);
+	// Setup user code.
+	next = alloc_guest_mem(&allocator, KVM_MAX_VCPU * KVM_PAGE_SIZE);
+	vm_set_user_memory_region(vmfd, slot++, KVM_MEM_READONLY, RISCV64_ADDR_USER_CODE, next.size, (uintptr_t)next.addr);
+	if (text_slot)
+		*text_slot = next.addr;
+	// Setup per-vCPU stacks.
+	next = alloc_guest_mem(&allocator, KVM_PAGE_SIZE);
+	vm_set_user_memory_region(vmfd, slot++, 0, RISCV64_ADDR_STACK_BASE, next.size, (uintptr_t)next.addr);
+	// Setup scratch memory for code generated at runtime.
+	next = alloc_guest_mem(&allocator, KVM_PAGE_SIZE);
+	vm_set_user_memory_region(vmfd, slot++, 0, RISCV64_ADDR_SCRATCH_CODE, next.size, (uintptr_t)next.addr);
+	// Setup exception vector table.
+	next = alloc_guest_mem(&allocator, KVM_PAGE_SIZE);
+	memcpy(next.addr, (void*)guest_unexp_trap, KVM_PAGE_SIZE);
+	vm_set_user_memory_region(vmfd, slot++, KVM_MEM_READONLY, RISCV64_ADDR_EXCEPTION_VECTOR, next.size, (uintptr_t)next.addr);
+	// Setup remaining RAM.
+	next = alloc_guest_mem(&allocator, allocator.size);
+	vm_set_user_memory_region(vmfd, slot++, 0, 0, next.size, (uintptr_t)next.addr);
+}
+
+static long syz_kvm_setup_syzos_vm(volatile long a0, volatile long a1)
+{
+	const int vmfd = a0;
+	void* host_mem = (void*)a1;
+
+	void* user_text_slot = NULL;
+	struct kvm_syz_vm* ret = (struct kvm_syz_vm*)host_mem;
+	host_mem = (void*)((uint64)host_mem + KVM_PAGE_SIZE);
+	setup_vm(vmfd, host_mem, &user_text_slot);
+	ret->vmfd = vmfd;
+	ret->next_cpu_id = 0;
+	ret->user_text = user_text_slot;
+	return (long)ret;
+}
+#endif
+
+#if SYZ_EXECUTOR || __NR_syz_kvm_add_vcpu
+// Set up CPU registers.
+static void reset_cpu_regs(int cpufd, int cpu_id, size_t text_size)
+{
+	// PC points to the relative offset of guest_main() within the guest code.
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_PC), executor_fn_guest_addr(guest_main));
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_SP), RISCV64_ADDR_STACK_BASE + KVM_PAGE_SIZE - 128);
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_TP), cpu_id);
+	// Pass parameters to guest_main().
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_A0), text_size);
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_A1), cpu_id);
+	// Set SSTATUS and MODE.
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_MODE), 1);
+	kvm_set_reg(cpufd, RISCV_CSR_REG(CSR_SSTATUS), SSTATUS_SPP | SSTATUS_SPIE);
+	// Set GP.
+	unsigned long current_gp = 0;
+	asm volatile("add %0, gp, zero"
+		     : "=r"(current_gp)
+		     :
+		     : "memory");
+	kvm_set_reg(cpufd, RISCV_CORE_REG(CORE_GP), current_gp);
+	// Set STVEC.
+	kvm_set_reg(cpufd, RISCV_CSR_REG(CSR_STVEC), RISCV64_ADDR_EXCEPTION_VECTOR);
+}
+
+static void install_user_code(int cpufd, void* user_text_slot, int cpu_id, const void* text, size_t text_size)
+{
+	if ((cpu_id < 0) || (cpu_id >= KVM_MAX_VCPU))
+		return;
+	if (!user_text_slot)
+		return;
+	if (text_size > KVM_PAGE_SIZE)
+		text_size = KVM_PAGE_SIZE;
+	void* target = (void*)((uint64)user_text_slot + (KVM_PAGE_SIZE * cpu_id));
+	memcpy(target, text, text_size);
+	reset_cpu_regs(cpufd, cpu_id, text_size);
+}
+
+static long syz_kvm_add_vcpu(volatile long a0, volatile long a1, volatile long a2, volatile long a3)
+{
+	struct kvm_syz_vm* vm = (struct kvm_syz_vm*)a0;
+	struct kvm_text* utext = (struct kvm_text*)a1;
+	const void* text = utext->text;
+	size_t text_size = utext->size;
+
+	if (!vm) {
+		errno = EINVAL;
+		return -1;
+	}
+	if (vm->next_cpu_id == KVM_MAX_VCPU) {
+		errno = ENOMEM;
+		return -1;
+	}
+	int cpu_id = vm->next_cpu_id;
+	int cpufd = ioctl(vm->vmfd, KVM_CREATE_VCPU, cpu_id);
+	if (cpufd == -1)
+		return -1;
+	// Only increment next_cpu_id if CPU creation succeeded.
+	vm->next_cpu_id++;
+	install_user_code(cpufd, vm->user_text, cpu_id, text, text_size);
+	return cpufd;
+}
+#endif
+
+#if SYZ_EXECUTOR || __NR_syz_kvm_assert_syzos_uexit
+static long syz_kvm_assert_syzos_uexit(volatile long a0, volatile long a1,
+				       volatile long a2)
+{
+#if !SYZ_EXECUTOR
+	int cpufd = (int)a0;
+#endif
+	struct kvm_run* run = (struct kvm_run*)a1;
+	uint64 expect = a2;
+
+	if (!run || (run->exit_reason != KVM_EXIT_MMIO) ||
+	    (run->mmio.phys_addr != RISCV64_ADDR_UEXIT)) {
+#if !SYZ_EXECUTOR
+		fprintf(stderr, "[SYZOS-DEBUG] Assertion Triggered on VCPU %d\n", cpufd);
+#endif
+		errno = EINVAL;
+		return -1;
+	}
+
+	uint64_t actual_code = ((uint64_t*)(run->mmio.data))[0];
+	if (actual_code != expect) {
+#if !SYZ_EXECUTOR
+		fprintf(stderr, "[SYZOS-DEBUG] Exit Code Mismatch on VCPU %d\n", cpufd);
+		fprintf(stderr, "   Expected: 0x%lx\n", (unsigned long)expect);
+		fprintf(stderr, "   Actual:   0x%lx\n",
+			(unsigned long)actual_code);
+#endif
+		errno = EDOM;
+		return -1;
+	}
+	return 0;
+}
+#endif
+
+#endif // EXECUTOR_COMMON_KVM_RISCV64_H