dma/hisi_acc: add data path operations

This commit adds data path ops for Hisilicon accelerator DMA driver.

Signed-off-by: Chengwen Feng <[email protected]>

Author: fengchengwen

doc/guides/dmadevs/hisi_acc.rst

@@ -53,3 +53,16 @@ Configuration requirements:
 * Only one ``vchan`` is supported per ``dmadev``.
 * Silent mode is not supported.
 * The transfer direction must be set to ``RTE_DMA_DIR_MEM_TO_MEM``.
+
+
+Device Datapath Capability and Limitation
+-----------------------------------------
+
+Support memory copy and fill operations.
+
+.. note::
+
+   Currently, the maximum size of the operation data is limited to 16MB-1B in the driver.
+   The device actually supports operations in a larger data size,
+   but the driver requires complex operations in the datapth.
+   If you have such requirement, please contact the maintainers.

drivers/dma/hisi_acc/hisi_acc_dmadev.c

@@ -8,6 +8,7 @@
 #include <sys/ioctl.h>
 
 #include <rte_byteorder.h>
+#include <rte_cycles.h>
 #include <rte_eal.h>
 #include <rte_io.h>
 #include <rte_kvargs.h>
@@ -79,6 +80,7 @@ hacc_dma_start(struct rte_dma_dev *dev)
 	if (hw->started) {
 		hw->ridx = 0;
 		hw->cridx = 0;
+		hw->stop_proc = 0;
 		return 0;
 	}
 
@@ -94,6 +96,7 @@ hacc_dma_start(struct rte_dma_dev *dev)
 	hw->cq_head = 0;
 	hw->cqs_completed = 0;
 	hw->cqe_vld = 1;
+	hw->stop_proc = 0;
 	hw->submitted = 0;
 	hw->completed = 0;
 	hw->errors = 0;
@@ -110,17 +113,33 @@ hacc_dma_start(struct rte_dma_dev *dev)
 static int
 hacc_dma_stop(struct rte_dma_dev *dev)
 {
+#define MAX_WAIT_MSEC	1000
+#define MAX_CPL_NUM	64
 	struct hacc_dma_dev *hw = dev->data->dev_private;
-
-	if ((*hw->sq_status != 0) || (*hw->cq_status != 0)) {
-		/* This indicates that the dev is abnormal. The correct error handling
-		 * is to close the dev (so that kernel module will perform error handling)
-		 * and apply for a new dev.
-		 * If an error code is returned here, the dev cannot be closed. Therefore,
-		 * zero is returned and an error trace is added.
-		 */
-		HACC_DMA_ERR(hw, "detect dev is abnormal!");
-		return 0;
+	uint32_t wait_msec = 0;
+
+	/* Flag stop processing new requests. */
+	hw->stop_proc = 1;
+	/* Currently, there is no method to notify the hardware to stop.
+	 * Therefore, the timeout mechanism is used to wait for the dataplane
+	 * to stop.
+	 */
+	while (hw->sq_head != hw->sq_tail && wait_msec++ < MAX_WAIT_MSEC) {
+		if ((*hw->sq_status != 0) || (*hw->cq_status != 0)) {
+			/* This indicates that the dev is abnormal. The correct error handling
+			 * is to close the dev (so that kernel module will perform error handling)
+			 * and apply for a new dev.
+			 * If an error code is returned here, the dev cannot be closed. Therefore,
+			 * zero is returned and an error trace is added.
+			 */
+			HACC_DMA_ERR(hw, "detect dev is abnormal!");
+			return 0;
+		}
+		rte_delay_ms(1);
+	}
+	if (hw->sq_head != hw->sq_tail) {
+		HACC_DMA_ERR(hw, "dev is still active!");
+		return -EBUSY;
 	}
 
 	return 0;
@@ -187,10 +206,11 @@ hacc_dma_dump(const struct rte_dma_dev *dev, FILE *f)
 {
 	struct hacc_dma_dev *hw = dev->data->dev_private;
 
-	fprintf(f, "  sqn: %u sq_status: %s cq_status: %s\n"
+	fprintf(f, "  sqn: %u sq_status: %s cq_status: %s stop_proc: %u\n"
 		"  sqe_size: %u sq_depth: %u sq_depth_mask: %u cq_depth: %u\n",
 		hw->sqn, (*hw->sq_status != 0) ? "ERR" : "OK",
 		(*hw->cq_status != 0) ? "ERR" : "OK",
+		hw->stop_proc,
 		hw->sqe_size, hw->sq_depth, hw->sq_depth_mask, hw->cq_depth);
 	fprintf(f, "  ridx: %u cridx: %u\n"
 		"  sq_head: %u sq_tail: %u cq_sq_head: %u avail_sqes: %u\n"
@@ -206,6 +226,280 @@ hacc_dma_dump(const struct rte_dma_dev *dev, FILE *f)
 	return 0;
 }
 
+static inline void
+hacc_dma_sq_doorbell(struct hacc_dma_dev *hw)
+{
+	uint64_t doorbell = (uint64_t)(hw->sqn & HACC_DMA_DOORBELL_SQN_MASK) |
+			    (HACC_DMA_DOORBELL_SQ_CMD << HACC_DMA_DOORBELL_CMD_SHIFT) |
+			    (((uint64_t)hw->sq_tail) << HACC_DMA_DOORBELL_IDX_SHIFT);
+	rte_io_wmb();
+	*(volatile uint64_t *)hw->doorbell_reg = doorbell;
+}
+
+static int
+hacc_dma_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
+	      uint32_t length, uint64_t flags)
+{
+	struct hacc_dma_dev *hw = dev_private;
+	struct hacc_dma_sqe *sqe = &hw->sqe[hw->sq_tail];
+
+	RTE_SET_USED(vchan);
+
+	if (unlikely(hw->stop_proc > 0))
+		return -EPERM;
+
+	if (unlikely(length > HACC_DMA_MAX_OP_SIZE)) {
+		hw->invalid_lens++;
+		return -EINVAL;
+	}
+
+	if (unlikely(*hw->sq_status != 0)) {
+		hw->io_errors++;
+		return -EIO;
+	}
+
+	if (hw->avail_sqes == 0) {
+		hw->qfulls++;
+		return -ENOSPC;
+	}
+
+	sqe->bd_type       = HACC_DMA_SQE_TYPE;
+	sqe->task_type     = HACC_DMA_TASK_TYPE;
+	sqe->task_type_ext = HACC_DMA_DATA_MEMCPY;
+	sqe->init_val      = 0;
+	sqe->addr_array    = src;
+	sqe->dst_addr      = dst;
+	sqe->data_size     = length;
+	sqe->dw0           = HACC_DMA_SVA_PREFETCH_EN;
+	sqe->wb_field      = 0;
+
+	hw->sq_tail = (hw->sq_tail + 1) & hw->sq_depth_mask;
+	hw->avail_sqes--;
+	hw->submitted++;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT)
+		hacc_dma_sq_doorbell(hw);
+
+	return hw->ridx++;
+}
+
+static int
+hacc_dma_fill(void *dev_private, uint16_t vchan, uint64_t pattern,
+	      rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+	struct hacc_dma_dev *hw = dev_private;
+	struct hacc_dma_sqe *sqe = &hw->sqe[hw->sq_tail];
+
+	RTE_SET_USED(vchan);
+
+	if (unlikely(hw->stop_proc > 0))
+		return -EPERM;
+
+	if (unlikely(length > HACC_DMA_MAX_OP_SIZE)) {
+		hw->invalid_lens++;
+		return -EINVAL;
+	}
+
+	if (unlikely(*hw->sq_status != 0)) {
+		hw->io_errors++;
+		return -EIO;
+	}
+
+	if (hw->avail_sqes == 0) {
+		hw->qfulls++;
+		return -ENOSPC;
+	}
+
+	sqe->bd_type       = HACC_DMA_SQE_TYPE;
+	sqe->task_type     = HACC_DMA_TASK_TYPE;
+	sqe->task_type_ext = HACC_DMA_DATA_MEMSET;
+	sqe->init_val      = pattern;
+	sqe->addr_array    = 0;
+	sqe->dst_addr      = dst;
+	sqe->data_size     = length;
+	sqe->dw0           = HACC_DMA_SVA_PREFETCH_EN;
+	sqe->wb_field      = 0;
+
+	hw->sq_tail = (hw->sq_tail + 1) & hw->sq_depth_mask;
+	hw->avail_sqes--;
+	hw->submitted++;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT)
+		hacc_dma_sq_doorbell(hw);
+
+	return hw->ridx++;
+}
+
+static int
+hacc_dma_submit(void *dev_private, uint16_t vchan)
+{
+	struct hacc_dma_dev *hw = dev_private;
+
+	RTE_SET_USED(vchan);
+
+	if (unlikely(*hw->sq_status != 0)) {
+		hw->io_errors++;
+		return -EIO;
+	}
+
+	hacc_dma_sq_doorbell(hw);
+
+	return 0;
+}
+
+static inline void
+hacc_dma_cq_doorbell(struct hacc_dma_dev *hw)
+{
+	uint64_t doorbell = (uint64_t)(hw->sqn & HACC_DMA_DOORBELL_SQN_MASK) |
+			    (HACC_DMA_DOORBELL_CQ_CMD << HACC_DMA_DOORBELL_CMD_SHIFT) |
+			    (((uint64_t)hw->cq_head) << HACC_DMA_DOORBELL_IDX_SHIFT);
+	rte_io_wmb();
+	*(volatile uint64_t *)hw->doorbell_reg = doorbell;
+}
+
+static inline void
+hacc_dma_scan_cq(struct hacc_dma_dev *hw)
+{
+	volatile struct hacc_dma_cqe *cqe;
+	struct hacc_dma_sqe *sqe;
+	uint16_t csq_head = hw->cq_sq_head;
+	uint16_t cq_head = hw->cq_head;
+	uint16_t count = 0;
+	uint64_t misc;
+
+	if (unlikely(*hw->cq_status != 0)) {
+		hw->io_errors++;
+		return;
+	}
+
+	while (count < hw->cq_depth) {
+		cqe = &hw->cqe[cq_head];
+		misc = cqe->misc;
+		misc = rte_le_to_cpu_64(misc);
+		if (RTE_FIELD_GET64(HACC_DMA_CQE_VALID_B, misc) != hw->cqe_vld)
+			break;
+
+		csq_head = RTE_FIELD_GET64(HACC_DMA_SQ_HEAD_MASK, misc);
+		if (unlikely(csq_head > hw->sq_depth_mask)) {
+			/**
+			 * Defensive programming to prevent overflow of the
+			 * status array indexed by csq_head. Only error logs
+			 * are used for prompting.
+			 */
+			HACC_DMA_ERR(hw, "invalid csq_head: %u!", csq_head);
+			count = 0;
+			break;
+		}
+		sqe = &hw->sqe[csq_head];
+		if (sqe->done_flag != HACC_DMA_TASK_DONE ||
+			sqe->err_type || sqe->ext_err_type || sqe->wtype) {
+			hw->status[csq_head] = RTE_DMA_STATUS_ERROR_UNKNOWN;
+		}
+
+		count++;
+		cq_head++;
+		if (cq_head == hw->cq_depth) {
+			hw->cqe_vld = !hw->cqe_vld;
+			cq_head = 0;
+		}
+	}
+
+	if (count == 0)
+		return;
+
+	hw->cq_head = cq_head;
+	hw->cq_sq_head = (csq_head + 1) & hw->sq_depth_mask;
+	hw->avail_sqes += count;
+	hw->cqs_completed += count;
+	if (hw->cqs_completed >= HACC_DMA_CQ_DOORBELL_PACE) {
+		hacc_dma_cq_doorbell(hw);
+		hw->cqs_completed = 0;
+	}
+}
+
+static inline uint16_t
+hacc_dma_calc_cpls(struct hacc_dma_dev *hw, const uint16_t nb_cpls)
+{
+	uint16_t cpl_num;
+
+	if (hw->cq_sq_head >= hw->sq_head)
+		cpl_num = hw->cq_sq_head - hw->sq_head;
+	else
+		cpl_num = hw->sq_depth_mask + 1 - hw->sq_head + hw->cq_sq_head;
+
+	if (cpl_num > nb_cpls)
+		cpl_num = nb_cpls;
+
+	return cpl_num;
+}
+
+static uint16_t
+hacc_dma_completed(void *dev_private,
+		   uint16_t vchan, const uint16_t nb_cpls,
+		   uint16_t *last_idx, bool *has_error)
+{
+	struct hacc_dma_dev *hw = dev_private;
+	uint16_t sq_head = hw->sq_head;
+	uint16_t cpl_num, i;
+
+	RTE_SET_USED(vchan);
+	hacc_dma_scan_cq(hw);
+
+	cpl_num = hacc_dma_calc_cpls(hw, nb_cpls);
+	for (i = 0; i < cpl_num; i++) {
+		if (hw->status[sq_head]) {
+			*has_error = true;
+			break;
+		}
+		sq_head = (sq_head + 1) & hw->sq_depth_mask;
+	}
+	*last_idx = hw->cridx + i - 1;
+	if (i > 0) {
+		hw->cridx += i;
+		hw->sq_head = sq_head;
+		hw->completed += i;
+	}
+
+	return i;
+}
+
+static uint16_t
+hacc_dma_completed_status(void *dev_private,
+			  uint16_t vchan, const uint16_t nb_cpls,
+			  uint16_t *last_idx, enum rte_dma_status_code *status)
+{
+	struct hacc_dma_dev *hw = dev_private;
+	uint16_t sq_head = hw->sq_head;
+	uint16_t cpl_num, i;
+
+	RTE_SET_USED(vchan);
+	hacc_dma_scan_cq(hw);
+
+	cpl_num = hacc_dma_calc_cpls(hw, nb_cpls);
+	for (i = 0; i < cpl_num; i++) {
+		status[i] = hw->status[sq_head];
+		hw->errors += !!status[i];
+		hw->status[sq_head] = 0;
+		sq_head = (sq_head + 1) & hw->sq_depth_mask;
+	}
+	*last_idx = hw->cridx + cpl_num - 1;
+	if (likely(cpl_num > 0)) {
+		hw->cridx += cpl_num;
+		hw->sq_head = sq_head;
+		hw->completed += cpl_num;
+	}
+
+	return cpl_num;
+}
+
+static uint16_t
+hacc_dma_burst_capacity(const void *dev_private, uint16_t vchan)
+{
+	const struct hacc_dma_dev *hw = dev_private;
+	RTE_SET_USED(vchan);
+	return hw->avail_sqes;
+}
+
 static const struct rte_dma_dev_ops hacc_dmadev_ops = {
 	.dev_info_get     = hacc_dma_info_get,
 	.dev_configure    = hacc_dma_configure,
@@ -297,6 +591,12 @@ hacc_dma_create(struct rte_uacce_device *uacce_dev, uint16_t queue_id)
 	dev->device = &uacce_dev->device;
 	dev->dev_ops = &hacc_dmadev_ops;
 	dev->fp_obj->dev_private = dev->data->dev_private;
+	dev->fp_obj->copy = hacc_dma_copy;
+	dev->fp_obj->fill = hacc_dma_fill;
+	dev->fp_obj->submit = hacc_dma_submit;
+	dev->fp_obj->completed = hacc_dma_completed;
+	dev->fp_obj->completed_status = hacc_dma_completed_status;
+	dev->fp_obj->burst_capacity = hacc_dma_burst_capacity;
 
 	hw = dev->data->dev_private;
 	hw->data = dev->data; /* make sure ACC_DMA_DEBUG/INFO/WARN/ERR was available. */