sunxi-fel: use RX DMA for FEL writes

The newer BROM FEL path copies received USB data byte-by-byte from the
MUSB FIFO. Even after switching the controller to high-speed mode, this
leaves large post-SPL DRAM writes far slower than the USB link can
support.

After SPL has initialized DRAM, install a parameterized RX DMA thunk for
writes into the known DRAM window. The thunk builds the MMU remap it
needs, hooks the BROM RX FIFO copy helper, and replaces it with a
MUSB/USBC endpoint-DMA receive path. It routes the active RX endpoint
DRQ through VEND0 before starting the internal DMA channel, so the FEL
wire protocol stays unchanged.

Keep the common thunk code in fel.c and put the SoC-specific hook,
translation-table and shadow-page addresses in the SoC table. Enable the
path for SoCs whose BROM dumps contain the same FIFO-copy helper ABI and
compatible MUSB register layout: A33, A64, H3, H5, A63, H6, H616, V853,
V5, A523 and A133.

Coalesce full high-speed packets into bounded RX DMA requests and use
larger progress chunks once the RX DMA thunk is installed. This reduces
device-side FIFO-copy overhead and host-side progress overhead for
large DRAM writes. Use a 192 KiB DMA request cap; 256 KiB requests can
time out while the host is writing.

Add the checked-in DMA thunk header to the sunxi-fel prerequisites and
thunk documentation. This makes the binary rebuild when the generated
header changes without making the sunxi-fel target regenerate thunk
headers.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>

Author: jameshilliard

Makefile

@@ -163,8 +163,10 @@ PROGRESS := progress.c progress.h
 SOC_INFO := soc_info.c soc_info.h
 FEL_LIB  := fel_lib.c fel_lib.h
 SPI_FLASH:= fel-spiflash.c fel-spiflash.h fel-remotefunc-spi-data-transfer.h
+FEL_THUNKS := thunks/fel-to-spl-thunk.h thunks/fel-to-secure-svc-smc-thunk.h
+FEL_THUNKS += thunks/fel-rx-dma-thunk.h
 
-sunxi-fel: fel.c fit_image.c thunks/fel-to-spl-thunk.h thunks/fel-to-secure-svc-smc-thunk.h $(PROGRESS) $(SOC_INFO) $(FEL_LIB) $(SPI_FLASH)
+sunxi-fel: fel.c fit_image.c $(FEL_THUNKS) $(PROGRESS) $(SOC_INFO) $(FEL_LIB) $(SPI_FLASH)
 	$(CC) $(HOST_CFLAGS) $(LIBUSB_CFLAGS) $(ZLIB_CFLAGS) $(LIBFDT_CFLAGS) $(LDFLAGS) -o $@ \
 		$(filter %.c,$^) $(LIBS) $(LIBUSB_LIBS) $(ZLIB_LIBS) $(LIBFDT_LIBS)
 

fel.c

@@ -37,6 +37,7 @@ static uint32_t uboot_entry = 0; /* entry point (address) of U-Boot */
 static uint32_t uboot_size  = 0; /* size of U-Boot binary */
 static bool enter_in_aarch64 = false;
 static bool fel_usb_high_speed_enabled = false;
+static bool fel_dram_ready = false;
 
 #define FEL_REENUM_TIMEOUT_MS		5000
 #define FEL_REENUM_RETRY_MS		100
@@ -109,6 +110,12 @@ typedef struct image_header {
 
 #define SID_STR_SIZE		36
 
+#define	DRAM_BASE		0x40000000
+#define	DRAM_SIZE		0x80000000
+#define	DRAM_END		(DRAM_BASE + DRAM_SIZE)
+
+static void aw_fel_install_rx_dma_thunk(feldev_handle *dev);
+
 static uint32_t get_le32(const void *ptr)
 {
 	uint32_t val;
@@ -184,6 +191,10 @@ double aw_write_buffer(feldev_handle *dev, void *buf, uint32_t offset,
 
 	double start = gettime();
 
+	if (fel_usb_high_speed_enabled && fel_dram_ready &&
+	    offset >= DRAM_BASE && offset < DRAM_END)
+		aw_fel_install_rx_dma_thunk(dev);
+
 	aw_fel_write_buffer(dev, buf, offset, len, progress);
 	return gettime() - start;
 }
@@ -410,8 +421,9 @@ static uint32_t fel_to_secure_svc_smc_thunk[] = {
 	#include "thunks/fel-to-secure-svc-smc-thunk.h"
 };
 
-#define	DRAM_BASE		0x40000000
-#define	DRAM_SIZE		0x80000000
+static uint32_t fel_rx_dma_thunk[] = {
+	#include "thunks/fel-rx-dma-thunk.h"
+};
 
 uint32_t aw_read_arm_cp_reg(feldev_handle *dev, soc_info_t *soc_info,
 			    uint32_t coproc, uint32_t opc1, uint32_t crn,
@@ -467,6 +479,40 @@ void fel_writel(feldev_handle *dev, uint32_t addr, uint32_t val)
 	fel_writel_n(dev, addr, &val, 1);
 }
 
+static void aw_fel_install_rx_dma_thunk(feldev_handle *dev)
+{
+	const fel_rx_dma_info *dma = &dev->soc_info->fel_rx_dma;
+	uint32_t params[6], param_offset;
+	size_t i;
+
+	if (dev->rx_dma_patched || !dma->thunk_addr)
+		return;
+
+	param_offset = fel_rx_dma_thunk[1];
+	if (param_offset % sizeof(*fel_rx_dma_thunk) ||
+	    param_offset + sizeof(params) > sizeof(fel_rx_dma_thunk))
+		pr_fatal("RX DMA thunk: bad parameter offset 0x%x\n",
+			 param_offset);
+
+	uint32_t thunk_buf[ARRAY_SIZE(fel_rx_dma_thunk)];
+
+	pr_info("Patching BROM RX FIFO copy to use DMA.\n");
+	for (i = 0; i < sizeof(thunk_buf) / sizeof(*thunk_buf); i++)
+		thunk_buf[i] = htole32(fel_rx_dma_thunk[i]);
+	params[0] = htole32(dma->l1_tt_addr);
+	params[1] = htole32(dma->l2_tt_addr);
+	params[2] = htole32(dma->brom_hook_addr);
+	params[3] = htole32(dma->brom_hook_shadow_addr);
+	params[4] = htole32(dev->soc_info->usb_musb_base);
+	params[5] = htole32(dma->dma_max_len);
+	memcpy((uint8_t *)thunk_buf + param_offset, params, sizeof(params));
+
+	aw_fel_write(dev, thunk_buf, dma->thunk_addr, sizeof(thunk_buf));
+
+	aw_fel_execute(dev, dma->thunk_addr + fel_rx_dma_thunk[0]);
+	dev->rx_dma_patched = true;
+}
+
 static bool aw_fel_switch_usb_high_speed(feldev_handle *dev)
 {
 	soc_info_t *soc_info = dev->soc_info;
@@ -952,7 +998,7 @@ void aw_restore_and_enable_mmu(feldev_handle *dev,
 	};
 
 	pr_info("Setting write-combine mapping for DRAM.\n");
-	for (i = (DRAM_BASE >> 20); i < ((DRAM_BASE + DRAM_SIZE) >> 20); i++) {
+	for (i = (DRAM_BASE >> 20); i < (DRAM_END >> 20); i++) {
 		/* Clear TEXCB bits */
 		tt[i] &= ~((7 << 12) | (1 << 3) | (1 << 2));
 		/* Set TEXCB to 00100 (Normal uncached mapping) */
@@ -1119,6 +1165,7 @@ uint32_t aw_fel_write_and_execute_spl(feldev_handle *dev, uint8_t *buf, size_t l
 	/* re-enable the MMU if it was enabled by BROM */
 	if (tt != NULL)
 		aw_restore_and_enable_mmu(dev, soc_info, tt);
+	fel_dram_ready = true;
 
 	return spl_len;
 }
@@ -1651,7 +1698,7 @@ void usage(const char *cmd) {
 		"	-h, --help			Print this usage summary and exit\n"
 		"	-v, --verbose			Verbose logging\n"
 		"	-p, --progress			\"write\" transfers show a progress bar\n"
-		"	    --no-high-speed		Disable high-speed USB\n"
+		"	    --no-high-speed		Disable high-speed USB and fast writes\n"
 		"	-l, --list			Enumerate all (USB) FEL devices and exit\n"
 		"	-d, --dev bus:devnum		Use specific USB bus and device number\n"
 		"	    --sid SID			Select device by SID key (exact match)\n"

fel_lib.c

@@ -70,19 +70,17 @@ static void usb_error(int rc, const char *caption, int exitcode)
  * timeout, and "slow" transfers take place at approx. 64 KiB/sec - so we can
  * expect the maximum chunk being transmitted within 8 seconds or less.
  */
-static const int AW_USB_MAX_BULK_SEND = 512 * 1024; /* 512 KiB per bulk request */
+static const int AW_USB_MAX_BULK_SEND = 512 * 1024;
+static const int AW_USB_PROGRESS_BULK_SEND = 128 * 1024;
+static const int AW_USB_FAST_PROGRESS_BULK_SEND = 1024 * 1024;
 
+/*
+ * Keep progress updates frequent enough for slow BROM PIO transfers. Once an
+ * RX-DMA thunk is installed, 1 MiB chunks avoid host-side progress overhead.
+ */
 static void usb_bulk_send(libusb_device_handle *usb, int ep, const void *data,
-			  size_t length, bool progress)
+			  size_t length, size_t max_chunk, bool progress)
 {
-	/*
-	 * With no progress notifications, we'll use the maximum chunk size.
-	 * Otherwise, it's useful to lower the size (have more chunks) to get
-	 * more frequent status updates. 128 KiB per request seem suitable.
-	 * (Worst case of "slow" transfers -> one update every two seconds.)
-	 */
-	size_t max_chunk = progress ? 128 * 1024 : AW_USB_MAX_BULK_SEND;
-
 	size_t chunk;
 	int rc, sent;
 	while (length > 0) {
@@ -187,7 +185,7 @@ static void aw_send_usb_request(feldev_handle *dev, int type, int length)
 	};
 	req.length2 = req.length;
 	usb_bulk_send(dev->usb->handle, dev->usb->endpoint_out,
-		      &req, sizeof(req), false);
+		      &req, sizeof(req), AW_USB_MAX_BULK_SEND, false);
 }
 
 static bool aw_send_usb_request_may_disconnect(feldev_handle *dev, int type,
@@ -239,9 +237,18 @@ static bool aw_usb_read_may_disconnect(feldev_handle *dev, void *data,
 static void aw_usb_write(feldev_handle *dev, const void *data, size_t len,
 			 bool progress)
 {
+	size_t max_chunk = AW_USB_MAX_BULK_SEND;
+
+	if (progress) {
+		if (dev->rx_dma_patched)
+			max_chunk = AW_USB_FAST_PROGRESS_BULK_SEND;
+		else
+			max_chunk = AW_USB_PROGRESS_BULK_SEND;
+	}
+
 	aw_send_usb_request(dev, AW_USB_WRITE, len);
 	usb_bulk_send(dev->usb->handle, dev->usb->endpoint_out,
-		      data, len, progress);
+		      data, len, max_chunk, progress);
 	aw_read_usb_response(dev);
 }
 

fel_lib.h

@@ -35,6 +35,7 @@ typedef struct {
 	soc_name_t soc_name;
 	soc_info_t *soc_info;
 	bool disconnected;
+	bool rx_dma_patched;
 } feldev_handle;
 
 /* list_fel_devices() will return an array of this type */

soc_info.c

@@ -25,6 +25,8 @@
 #include <stdio.h>
 #include <string.h>
 
+#define FEL_RX_DMA_MAX_LEN	0x00030000
+
 /*
  * The FEL code from BROM in A10/A13/A20 sets up two stacks for itself. One
  * at 0x2000 (and growing down) for the IRQ handler. And another one at 0x7000
@@ -390,6 +392,14 @@ soc_info_t soc_info_table[] = {
 		.sid_sections = generic_2k_sid_maps,
 		.usb_musb_base= 0x01c19000,
 		.fel_endpoint_table_addr = 0x00006ed8,
+		.fel_rx_dma = {
+			.thunk_addr = 0x0004e000,
+			.l1_tt_addr = 0x00048000,
+			.l2_tt_addr = 0x0004c000,
+			.brom_hook_addr = 0xffff2588,
+			.brom_hook_shadow_addr = 0x0004d000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.watchdog     = &wd_h3_compat,
 	},{
 		.soc_id       = 0x1689, /* Allwinner A64 */
@@ -405,6 +415,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x017000A0,
 		.usb_musb_base= 0x01c19000,
 		.fel_endpoint_table_addr = 0x00016ecc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x0002c000,
+			.l2_tt_addr = 0x00031000,
+			.brom_hook_addr = 0x00002670,
+			.brom_hook_shadow_addr = 0x00030000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		/* Check L.NOP in the OpenRISC reset vector */
 		.needs_smc_workaround_if_zero_word_at_addr = 0x40004,
 		.smc_workaround = SMC_WORKAROUND_DIRECT_SMC,
@@ -456,6 +474,14 @@ soc_info_t soc_info_table[] = {
 		.sid_sections = h3_sid_maps,
 		.usb_musb_base= 0x01c19000,
 		.fel_endpoint_table_addr = 0x00006ecc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x0000e000,
+			.l1_tt_addr = 0x00008000,
+			.l2_tt_addr = 0x0000c000,
+			.brom_hook_addr = 0xffff2680,
+			.brom_hook_shadow_addr = 0x0000d000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		/* Check L.NOP in the OpenRISC reset vector */
 		.needs_smc_workaround_if_zero_word_at_addr = 0x40004,
 		.smc_workaround = SMC_WORKAROUND_DIRECT_SMC,
@@ -497,6 +523,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x017000A0,
 		.usb_musb_base= 0x01c19000,
 		.fel_endpoint_table_addr = 0x00016ecc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x0002c000,
+			.l2_tt_addr = 0x00031000,
+			.brom_hook_addr = 0x0000205c,
+			.brom_hook_shadow_addr = 0x00030000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		/* Check L.NOP in the OpenRISC reset vector */
 		.needs_smc_workaround_if_zero_word_at_addr = 0x40004,
 		.smc_workaround = SMC_WORKAROUND_DIRECT_SMC,
@@ -526,6 +560,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x09010040,
 		.usb_musb_base= 0x05100000,
 		.fel_endpoint_table_addr = 0x00026ecc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x00038000,
+			.l2_tt_addr = 0x0003d000,
+			.brom_hook_addr = 0x000021ac,
+			.brom_hook_shadow_addr = 0x0003c000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.watchdog     = &wd_h6_compat,
 	},{
 		.soc_id       = 0x1728, /* Allwinner H6 */
@@ -541,6 +583,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x09010040,
 		.usb_musb_base= 0x05100000,
 		.fel_endpoint_table_addr = 0x00026ecc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x00038000,
+			.l2_tt_addr = 0x0003d000,
+			.brom_hook_addr = 0x00002300,
+			.brom_hook_shadow_addr = 0x0003c000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		/* Check L.NOP in the OpenRISC reset vector */
 		.needs_smc_workaround_if_zero_word_at_addr = 0x100004,
 		.smc_workaround = SMC_WORKAROUND_DIRECT_SMC,
@@ -585,6 +635,14 @@ soc_info_t soc_info_table[] = {
 		.ver_reg      = 0x03000024,
 		.usb_musb_base= 0x05100000,
 		.fel_endpoint_table_addr = 0x000550f0,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x0004c000,
+			.l2_tt_addr = 0x00051000,
+			.brom_hook_addr = 0x0000a17c,
+			.brom_hook_shadow_addr = 0x00050000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.needs_smc_workaround_if_zero_word_at_addr = 0x03006240,
 		.secure_boot_fuse_offset = 0xa0,
 		.smc_workaround = SMC_WORKAROUND_SECURE_SVC_SMC_THUNK,
@@ -617,6 +675,14 @@ soc_info_t soc_info_table[] = {
 		.sid_sections = generic_2k_sid_maps,
 		.usb_musb_base= 0x04100000,
 		.fel_endpoint_table_addr = 0x000402e8,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x00038000,
+			.l2_tt_addr = 0x0003d000,
+			.brom_hook_addr = 0x0000aff0,
+			.brom_hook_shadow_addr = 0x0003c000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.icache_fix   = true,
 		.watchdog     = &wd_v853_compat,
 	},{
@@ -645,6 +711,14 @@ soc_info_t soc_info_table[] = {
 		.sid_sections = generic_2k_sid_maps,
 		.usb_musb_base= 0x05100000,
 		.fel_endpoint_table_addr = 0x00026efc,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x00038000,
+			.l2_tt_addr = 0x0003d000,
+			.brom_hook_addr = 0x000036ec,
+			.brom_hook_shadow_addr = 0x0003c000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.watchdog     = &wd_h6_compat,
 	},{
 		.soc_id       = 0x1890, /* Allwinner A523 */
@@ -660,6 +734,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x08000040,
 		.usb_musb_base= 0x04100000,
 		.fel_endpoint_table_addr = 0x0005e6ec,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00041000,
+			.l1_tt_addr = 0x00050000,
+			.l2_tt_addr = 0x00055000,
+			.brom_hook_addr = 0x00017574,
+			.brom_hook_shadow_addr = 0x00054000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.icache_fix   = true,
 		.watchdog     = &wd_a523_compat,
 	},{
@@ -676,6 +758,14 @@ soc_info_t soc_info_table[] = {
 		.rvbar_reg    = 0x08100040,
 		.usb_musb_base= 0x05100000,
 		.fel_endpoint_table_addr = 0x00041ff0,
+		.fel_rx_dma = {
+			.thunk_addr = 0x00022000,
+			.l1_tt_addr = 0x00038000,
+			.l2_tt_addr = 0x0003d000,
+			.brom_hook_addr = 0x0000b698,
+			.brom_hook_shadow_addr = 0x0003c000,
+			.dma_max_len = FEL_RX_DMA_MAX_LEN,
+		},
 		.needs_smc_workaround_if_zero_word_at_addr = 0x100004,
 		.smc_workaround = SMC_WORKAROUND_DIRECT_SMC,
 		.watchdog     = &wd_h6_compat,

soc_info.h

@@ -70,6 +70,15 @@ typedef struct {
 	uint32_t	size_bits;
 } sid_section;
 
+typedef struct {
+	uint32_t           thunk_addr;
+	uint32_t           l1_tt_addr;
+	uint32_t           l2_tt_addr;
+	uint32_t           brom_hook_addr;
+	uint32_t           brom_hook_shadow_addr;
+	uint32_t           dma_max_len;
+} fel_rx_dma_info;
+
 #define SID_SECTION(_name, _offset, _size_bits) {	\
 	.name = _name,					\
 	.offset = _offset,				\
@@ -149,6 +158,7 @@ typedef struct {
 	uint32_t           ver_reg;      /* MMIO address of "Version Register" */
 	uint32_t           usb_musb_base;/* base address of the USB OTG controller */
 	uint32_t           fel_endpoint_table_addr; /* BROM FEL endpoint table */
+	fel_rx_dma_info    fel_rx_dma;   /* BROM RX FIFO copy DMA patch */
 	const watchdog_info *watchdog;   /* Used for reset */
 	bool               sid_fix;      /* Use SID workaround (read via register) */
 	/* Use I$ workaround (disable I$ before first write to prevent stale thunk */

sunxi-fel.1

@@ -42,7 +42,7 @@ Enable verbose logging.
 .sp
 .B \-\-no-high-speed
 .RS 4
-Disable automatic high-speed USB switching.
+Disable automatic high-speed USB switching and post-SPL fast DRAM writes.
 .RE
 .sp
 .B \-l, \-\-list