Add USB UAC2 Audio IF to Zephyr Firmware

Author: malarisch

FPGA/boards/altera/cyc2/cyc2.qsf

@@ -61,6 +61,10 @@ set_location_assignment PIN_39 -to spictrl_cs
 set_location_assignment PIN_37 -to spictrl_irq
 set_location_assignment PIN_43 -to spictrl_miso
 set_location_assignment PIN_40 -to spictrl_mosi
+set_location_assignment PIN_198 -to esp_bclk
+set_location_assignment PIN_199 -to esp_fsclk
+set_location_assignment PIN_201 -to esp_din
+set_location_assignment PIN_203 -to esp_dout
 set_location_assignment PIN_89 -to c2_resetn
 set_location_assignment PIN_35 -to mdc
 set_location_assignment PIN_34 -to mdio

FPGA/boards/altera/cyc2/cyc2top.vhd

@@ -8,7 +8,7 @@ use work.audioclks_pkg.all;
 use work.system_cfg_pkg.all;
 ENTITY cyc2top IS
 	generic (
-		syscfg: t_global_system_cfg := global_system_cfg_lo
+		syscfg: t_global_system_cfg := global_system_cfg_rn2io
 	);
 	PORT 
 	(
@@ -17,6 +17,10 @@ ENTITY cyc2top IS
 
         fs_adc: OUT STD_LOGIC; -- lrclk
 		  fs_dac: OUT STD_LOGIC; -- lrclk
+		  esp_fsclk : OUT STD_LOGIC;
+		  esp_bclk : OUT STD_LOGIC;
+		  esp_din : IN STD_LOGIC;
+		  esp_dout : OUT std_logic;
 		  fs_ext : OUT STD_LOGIC;
         tdm_out: OUT STD_LOGIC_VECTOR(3 downto 0); -- tdm out
         bclk_adc: OUT STD_LOGIC; -- bclk
@@ -76,14 +80,18 @@ signal mclk_reg : std_logic;
 signal dbg_mac_tx_clk_o : std_logic;
 
 begin
-	tdm_in_reg <= tdm_in(syscfg.AUDIO_CONFIG.TX_AD_CFG.TDM_PINS - 1 downto 0);
+	--tdm_in_reg <= tdm_in(syscfg.AUDIO_CONFIG.TX_AD_CFG.TDM_PINS - 1 downto 0);
 	INIT_DONE <= '1';
     tx_err_o <= (others => '0');
-	tdm_out(syscfg.AUDIO_CONFIG.RX_DA_CFG.TDM_PINS - 1 downto 0) <= tdm_out_reg;
+	--tdm_out(syscfg.AUDIO_CONFIG.RX_DA_CFG.TDM_PINS - 1 downto 0) <= tdm_out_reg;
+	esp_dout <= tdm_out_reg(0);
+	tdm_in_reg(0) <= esp_din;
 	mclk <= mclk_reg;
 	 phy_rstn_o <= '1';
 	 bclk_adc <= not bclk;
 	 bclk_dac <= not bclk;
+	 esp_bclk <= not bclk;
+	 esp_fsclk <= fs;
 	 txen(0) <= phy_txen;
 	 phy0tx <= phy_tx;
 	 phy1tx <= phy_tx;

FPGA/packages/audioclks_pkg.vhd

@@ -61,6 +61,13 @@ package audioclks_pkg is
         fs_high_is_channel_2 => true,
         tdm_channels => 2
     );
+    constant i2s_esp_config : t_audio_clock_io_cfg := (
+        data_is_valid_on_rising_bclk_edge => true,
+        bits_are_right_shifted_to_fs => true,
+        fs_is_one_bclk_high => false,
+        fs_high_is_channel_2 => true,
+        tdm_channels => 2
+    );
     constant i2s_lj_dac_config : t_audio_clock_io_cfg := (
         data_is_valid_on_rising_bclk_edge => true,
         bits_are_right_shifted_to_fs => false,

FPGA/packages/system_cfg_pkg.vhd

@@ -113,6 +113,13 @@ package system_cfg_pkg is
         TDM_PINS => 1,
         ADDA_CFG => i2s_adc_config
     );
+    constant two_i2s_inputs_bclkinv : t_audio_cfg := (
+        MAX_STREAMS => 2,
+        BUFFER_DEPTH => 64,
+        CHANNELS => 2,
+        TDM_PINS => 1,
+        ADDA_CFG => i2s_esp_config
+    );
     constant four_i2s_outputs : t_audio_cfg := (
         MAX_STREAMS => 8,
         BUFFER_DEPTH => 256,
@@ -143,6 +150,14 @@ package system_cfg_pkg is
         RX_DA_CFG => four_i2s_outputs,
         TX_AD_CFG => disable_audio_path
     );
+    constant audio_config_2io : t_global_audio_cfg := (
+        MCLK_SPEED => audio_clock_24_57,
+        BCLK_SPEED => audio_clock_03_07,
+        USE_PARALLEL_INTERFACE => false,
+        PARALLEL_BYTE_DEPTH => 3,
+        RX_DA_CFG => two_i2s_inputs_bclkinv,
+        TX_AD_CFG => two_i2s_inputs_bclkinv
+    );
     constant audio_config_mi : t_global_audio_cfg := (
         MCLK_SPEED => audio_clock_24_57,
         BCLK_SPEED => audio_clock_03_07,
@@ -182,6 +197,17 @@ package system_cfg_pkg is
         ENABLE_METERING => false,
         PTP_MOVING_AVERAGE_DEPTH => 4
     );
+    constant global_system_cfg_rn2io : t_global_system_cfg := (
+        CLK_IN_SPEED => 25,
+        SOC_TYPE => LITEX_SPIBONE,
+        PLATFORM => ALTERA,
+        PHY_CONFIG => std_lxt_cfg,
+        AUDIO_CONFIG => audio_config_2io,
+        STATIC_PTP_CONFIG => true,
+        PTP_IN_SOFTWARE => true,
+        ENABLE_METERING => false,
+        PTP_MOVING_AVERAGE_DEPTH => 4
+    );
     constant global_system_cfg_cyc : t_global_system_cfg := (
         CLK_IN_SPEED => 12,
         SOC_TYPE => LITEX_VEXRISCV_SDRAM,

external/zephyr

@@ -239,6 +239,20 @@ elseif(CONFIG_FPGA_HAL_LITEX OR CONFIG_FPGA_HAL_SPI OR CONFIG_FPGA_HAL_MOCK)
       target_sources(app PRIVATE src/nmos/nmos_json.c)
     endif()
     target_sources_ifdef(CONFIG_SD_CONFIG         app PRIVATE src/sd_config.c)
+    # USB Audio Class 2 <-> FPGA I2S bridge (ESP32-S3 external MCU only).
+    # usb_audio_i2s.c drives I2S0 + GDMA through the ESP HAL directly (the
+    # Zephyr I2S driver cannot stream gaplessly); the I2S HAL sources are
+    # only compiled by the hal_espressif module when CONFIG_I2S_ESP32 is
+    # set, so pull them in here.
+    if(CONFIG_AES67_USB_AUDIO)
+      target_sources(app PRIVATE src/usb_audio.c src/usb_audio_i2s.c)
+      # register dump in `usbaudio dwc2` (bring-up diagnostics)
+      target_include_directories(app PRIVATE ${ZEPHYR_BASE}/drivers/usb/common)
+      target_sources(app PRIVATE
+        ${ZEPHYR_HAL_ESPRESSIF_MODULE_DIR}/components/esp_hal_i2s/i2s_hal.c
+        ${ZEPHYR_HAL_ESPRESSIF_MODULE_DIR}/components/hal/hal_utils.c
+      )
+    endif()
     target_sources_ifdef(CONFIG_FLASH_CONFIG      app PRIVATE src/flash_config.c)
     if(CONFIG_THREAD_RUNTIME_STATS AND CONFIG_SHELL)
       target_sources(app PRIVATE src/perf_stats.c)

soc_firmware/app/CMakeLists.txt

@@ -53,6 +53,75 @@ config APP_SPI_BRINGUP
 	  (HTTP / RTSP / mDNS / PTP BMC) is the default for every HAL
 	  backend that has networking, including SPI.
 
+menu "USB audio interface (UAC2 <-> FPGA I2S)"
+
+config AES67_USB_AUDIO
+	bool "Expose the FPGA audio port as a USB Audio Class 2 device"
+	default n
+	depends on FPGA_HAL_SPI
+	depends on USB_DEVICE_STACK_NEXT && USBD_AUDIO2_CLASS
+	depends on SOC_SERIES_ESP32S3
+	help
+	  Experimental "AES67 USB audio interface": the ESP32-S3's USB-OTG
+	  core enumerates as a UAC2 device (2 in / 2 out, 48 kHz, 24 bit)
+	  and the samples are shovelled between the USB isochronous
+	  endpoints and an I2S port on which the ESP32 is the *target*
+	  (slave) of the FPGA's PTP-disciplined media clock. Implicit
+	  feedback keeps the host's rate locked to the FPGA clock. See
+	  soc_firmware/docs/usb-audio-plan.md and src/usb_audio.c.
+
+	  Needs the zephyr,uac2 node + i2s0 + usb_otg from the board overlay
+	  and moves the console off the USB Serial/JTAG port (shared PHY).
+
+if AES67_USB_AUDIO
+
+config AES67_USB_AUDIO_VID
+	hex "USB vendor ID"
+	default 0x1209
+	help
+	  0x1209 is pid.codes; PIDs 0x0001-0x0010 there are reserved for
+	  testing/private use, which is what this experiment is. Replace
+	  before shipping anything.
+
+config AES67_USB_AUDIO_PID
+	hex "USB product ID"
+	default 0x0001
+
+config AES67_USB_AUDIO_MANUFACTURER
+	string "USB manufacturer string"
+	default "AES67 project"
+
+config AES67_USB_AUDIO_PRODUCT
+	string "USB product string suffix"
+	default "AES67USB"
+	help
+	  The USB product string is "<hostname> <suffix>", the hostname being
+	  the same one the device uses on the network (aes67_config). The
+	  suffix alone is used when no hostname is configured. Note that
+	  desktops resolve the model name from the USB ID database first;
+	  with a pid.codes test PID they display "pid.codes Test PID"
+	  regardless of this string.
+
+config AES67_USB_AUDIO_MAX_POWER
+	int "bMaxPower (2 mA units)"
+	default 250
+	help
+	  Bus power the configuration descriptor asks for. 250 = 500 mA:
+	  the DevKitC + FPGA board are powered from this port in the
+	  bench setup.
+
+config AES67_USB_AUDIO_AUTOSTART
+	bool "Enable the USB device at boot"
+	default y
+	help
+	  Bring the USB device up at the end of main(). Off = the I2S/USB
+	  path stays idle until 'usbaudio start' is issued from the shell
+	  (bring-up aid).
+
+endif # AES67_USB_AUDIO
+
+endmenu
+
 menu "RAVENNA RTSP Server/Client"
 
 config RTSP

soc_firmware/app/Kconfig

@@ -19,7 +19,20 @@
  *   ADDA nRST = GPIO14 (active low — low holds the card in reset;
  *                       shared with the display controller)
  *
- * Console/shell: built-in USB Serial/JTAG (usb_serial), not uart0.
+ * Console/shell: uart0 (GPIO43 TX / GPIO44 RX, the DevKitC's "UART"
+ * USB connector). The built-in USB Serial/JTAG controller shares the
+ * chip's single internal USB PHY with the USB-OTG core; enabling the USB
+ * audio device (usb_otg) hands the PHY to OTG, so the "USB" connector is
+ * the audio interface and can no longer carry the console.
+ *
+ * USB audio (UAC2) <-> FPGA I2S, ESP32 is I2S *target* (slave) of the
+ * FPGA's media clock (BCLK = 64 fs, WS = fs, 24-bit in 32-bit slots):
+ *   BCLK = GPIO40 (in,  FPGA bclk_dac / bclk_adc)
+ *   WS   = GPIO41 (in,  FPGA fs_dac / fs_adc)
+ *   DIN  = GPIO42 (in,  FPGA tdm_out[0]  -> USB record)
+ *   DOUT = GPIO39 (out, FPGA tdm_in[0]   <- USB playback)
+ * GPIO39..42 are the chip's own JTAG pads (unused: FPGA JTAG is bit-banged
+ * on GPIO4..7, the ESP itself is debugged over USB-JTAG/UART).
  *
  * Display controller (LEDs/buttons/7-segment, 115200 8N1):
  *   TX = GPIO17, RX = GPIO18 (board default uart1 pinmux)
@@ -33,6 +46,8 @@
  */
 
 #include <zephyr/dt-bindings/gpio/gpio.h>
+#include <zephyr/dt-bindings/pinctrl/esp32s3-pinctrl.h>
+#include <zephyr/dt-bindings/usb/audio.h>
 
 &psram0 {
 	size = <DT_SIZE_M(8)>;
@@ -55,6 +70,47 @@
 		};
 	};
 
+	/* I2S0 as full-duplex target: BCLK and WS are inputs only, so they
+	 * are muxed with explicit ESP32_PINMUX(pin, <in-signal>, ESP_NOSIG)
+	 * instead of the stock I2S0_O_BCK_GPIOnn macros, which also connect
+	 * the peripheral's output signal and enable the pad driver — that
+	 * would fight the FPGA's clock outputs. The TX (O_) and RX (I_)
+	 * halves both get the same pads (the GPIO matrix fans one pad out
+	 * to several input signals); the driver additionally sets
+	 * sig_loopback so RX follows TX's clock inputs. */
+	i2s0_fpga: i2s0_fpga {
+		group1 {
+			pinmux = <ESP32_PINMUX(40, ESP_I2S0O_BCK_IN, ESP_NOSIG)>,
+				 <ESP32_PINMUX(40, ESP_I2S0I_BCK_IN, ESP_NOSIG)>,
+				 <ESP32_PINMUX(41, ESP_I2S0O_WS_IN, ESP_NOSIG)>,
+				 <ESP32_PINMUX(41, ESP_I2S0I_WS_IN, ESP_NOSIG)>,
+				 <I2S0_I_SD_GPIO42>;
+		};
+		group2 {
+			pinmux = <I2S0_O_SD_GPIO39>;
+			output-low;
+		};
+	};
+};
+
+/* I2S0 <-> FPGA audio port. usb_audio_i2s.c programs the controller and
+ * GDMA channel 0 through the ESP HAL (Zephyr's I2S driver restarts the DMA
+ * per block and drops samples in the gap); it only takes reg, clocks and
+ * the pinctrl from this node. The node stays disabled so the Zephyr driver
+ * is never instantiated, and `dma` stays disabled as in the board DTS. */
+&i2s0 {
+	pinctrl-0 = <&i2s0_fpga>;
+	pinctrl-names = "default";
+};
+
+/* USB-OTG device controller (already labelled zephyr_udc0 / okay in the
+ * board DTS) and the USB Audio Class 2 function on top of it. */
+&usb_otg {
+	status = "okay";
+};
+
+&usb_serial {
+	status = "disabled";
 };
 
 &spi2 {
@@ -101,19 +157,9 @@ i2c2: &i2c0 {
 	pinctrl-names = "default";
 };
 
-/* Console + shell on the ESP32-S3's built-in USB Serial/JTAG controller
- * (the "USB" port of the DevKitC, CDC-ACM towards the host) instead of
- * uart0 / the UART-bridge port. Zephyr derives ESP_CONSOLE_USB_SERIAL_JTAG
- * and the ROM console device number from this chosen node, so no extra
- * Kconfig is needed. uart0 stays enabled as a plain peripheral. */
-&usb_serial {
-	status = "okay";
-};
-
 / {
+	/* Console + shell stay on the board default, uart0 (see header). */
 	chosen {
-		zephyr,console = &usb_serial;
-		zephyr,shell-uart = &usb_serial;
 		zephyr,fpga-spi = &fpga_spi;
 	};
 
@@ -145,3 +191,79 @@ i2c2: &i2c0 {
 		gpios = <&gpio0 14 GPIO_ACTIVE_LOW>;
 	};
 };
+
+/* USB Audio Class 2 function: 2 in / 2 out, 48 kHz, 24 bit in 3-byte
+ * subslots (294 B per Full-Speed packet; 4-byte subslots would need a
+ * 392 B RxFIFO, more than the ESP32-S3's DWC2 can spare next to the IN
+ * FIFO, see CONFIG_UDC_DWC2_RXFIFO_MAX_PERCENTAGE). The clock is the FPGA's
+ * PTP-disciplined media clock: external, free-running, not SOF-locked ->
+ * asynchronous endpoints. Rate matching uses implicit feedback: the IN
+ * stream carries exactly as many frames per SOF as I2S delivered and the
+ * host derives the OUT packet size from that; both directions share the
+ * clock, so the playback queue stays level. Entity IDs are the child
+ * index + 1 (UAC2_ENTITY_ID()), the order below is therefore significant. */
+/ {
+	uac2_fpga: usb_audio2 {
+		compatible = "zephyr,uac2";
+		status = "okay";
+		full-speed;
+		audio-function = <AUDIO_FUNCTION_IO_BOX>;
+
+		uac_aclk: aclk {
+			compatible = "zephyr,uac2-clock-source";
+			clock-type = "external";
+			frequency-control = "read-only";
+			sampling-frequencies = <48000>;
+		};
+
+		/* Host -> device (USB playback) ... */
+		usb_out_terminal: usb_out_terminal {
+			compatible = "zephyr,uac2-input-terminal";
+			clock-source = <&uac_aclk>;
+			terminal-type = <USB_TERMINAL_STREAMING>;
+			front-left;
+			front-right;
+		};
+
+		/* ... leaves through the I2S port into the FPGA (AES67 TX). */
+		fpga_tx_terminal: fpga_tx {
+			compatible = "zephyr,uac2-output-terminal";
+			data-source = <&usb_out_terminal>;
+			clock-source = <&uac_aclk>;
+			terminal-type = <EXTERNAL_TERMINAL_DIGITAL_AUDIO_INTERFACE>;
+		};
+
+		/* FPGA (AES67 RX) enters through the I2S port ... */
+		fpga_rx_terminal: fpga_rx {
+			compatible = "zephyr,uac2-input-terminal";
+			clock-source = <&uac_aclk>;
+			terminal-type = <EXTERNAL_TERMINAL_DIGITAL_AUDIO_INTERFACE>;
+			front-left;
+			front-right;
+		};
+
+		/* ... and goes to the host (USB record). */
+		usb_in_terminal: usb_in_terminal {
+			compatible = "zephyr,uac2-output-terminal";
+			data-source = <&fpga_rx_terminal>;
+			clock-source = <&uac_aclk>;
+			terminal-type = <USB_TERMINAL_STREAMING>;
+		};
+
+		as_iso_out: out_interface {
+			compatible = "zephyr,uac2-audio-streaming";
+			linked-terminal = <&usb_out_terminal>;
+			implicit-feedback;
+			subslot-size = <3>;
+			bit-resolution = <24>;
+		};
+
+		as_iso_in: in_interface {
+			compatible = "zephyr,uac2-audio-streaming";
+			linked-terminal = <&usb_in_terminal>;
+			implicit-feedback;
+			subslot-size = <3>;
+			bit-resolution = <24>;
+		};
+	};
+};