diff --git a/src/virtio/virtio_9p.c b/src/virtio/virtio_9p.c index e2e914666..811953d8b 100644 --- a/src/virtio/virtio_9p.c +++ b/src/virtio/virtio_9p.c @@ -56,7 +56,7 @@ static u32 v9p_request(virtio_9p v9p, u64 req_phys, u32 req_len, u64 resp_phys, vqfinish finish = closure(v9p->general, v9p_req_complete, ctx, &ret_len); if (finish == INVALID_ADDRESS) return 0; - vqmsg m = allocate_vqmsg(v9p->vq); + vqmsg m = allocate_vqmsg(v9p->vq, 2); if (m == INVALID_ADDRESS) { deallocate_closure(finish); return 0; @@ -431,7 +431,7 @@ int v9p_readdir(void *priv, u32 fid, u64 offset, void *buf, u32 count, u32 *ret_ s = -ENOMEM; goto out; } - vqmsg m = allocate_vqmsg(v9p->vq); + vqmsg m = allocate_vqmsg(v9p->vq, 3); if (m == INVALID_ADDRESS) { deallocate_closure(finish); s = -ENOMEM; @@ -715,7 +715,7 @@ void v9p_read(void *priv, u32 fid, u64 offset, u32 count, void *dest, status_han s = timm("result", "failed to allocate vqfinish"); goto dealloc_req; } - vqmsg m = allocate_vqmsg(v9p->vq); + vqmsg m = allocate_vqmsg(v9p->vq, 3); if (m == INVALID_ADDRESS) { s = timm("result", "failed to allocate vqmsg"); deallocate_closure(finish); @@ -779,7 +779,7 @@ void v9p_write(void *priv, u32 fid, u64 offset, u32 count, void *src, status_han s = timm("result", "failed to allocate vqfinish"); goto dealloc_req; } - vqmsg m = allocate_vqmsg(v9p->vq); + vqmsg m = allocate_vqmsg(v9p->vq, 3); if (m == INVALID_ADDRESS) { s = timm("result", "failed to allocate vqmsg"); deallocate_closure(finish); diff --git a/src/virtio/virtio_balloon.c b/src/virtio/virtio_balloon.c index 221ec1431..a1eedd866 100644 --- a/src/virtio/virtio_balloon.c +++ b/src/virtio/virtio_balloon.c @@ -173,7 +173,7 @@ static u64 virtio_balloon_inflate(u64 n_balloon_pages) break; } - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, bp->phys, sizeof(bp->addrs), false); vqfinish c = closure(virtio_balloon.general, inflate_complete, bp); @@ -262,7 +262,7 @@ static u64 virtio_balloon_deflate(u64 n_balloon_pages, boolean sync) if (work == INVALID_ADDRESS) break; } - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); if (m == INVALID_ADDRESS) { if (!sync) deallocate(h, work, sizeof(*work)); @@ -394,7 +394,7 @@ closure_func_basic(vqfinish, void, virtio_balloon_enqueue_stats, write_stat(VIRTIO_BALLOON_S_HTLB_PGALLOC, 0); write_stat(VIRTIO_BALLOON_S_HTLB_PGFAIL, 0); - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, virtio_balloon.stats_phys, sizeof(struct virtio_balloon_stat) * virtio_balloon.next_tag, false); @@ -410,7 +410,7 @@ static void virtio_balloon_init_statsq(void) vqfinish c = closure_func(virtio_balloon.general, vqfinish, virtio_balloon_enqueue_stats); assert(c != INVALID_ADDRESS); - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, virtio_balloon.stats_phys, 8 /* arbitrary; zero-len queue not allowed */, false); diff --git a/src/virtio/virtio_internal.h b/src/virtio/virtio_internal.h index ebb8f79d2..171dc9cbc 100644 --- a/src/virtio/virtio_internal.h +++ b/src/virtio/virtio_internal.h @@ -130,9 +130,9 @@ void virtqueue_set_polling(virtqueue vq, boolean enable); typedef struct vqmsg *vqmsg; -vqmsg allocate_vqmsg(virtqueue vq); +vqmsg allocate_vqmsg(virtqueue vq, int num_desc); void deallocate_vqmsg(virtqueue vq, vqmsg m); -void vqmsg_push(virtqueue vq, vqmsg m, u64 phys_addr, u32 len, boolean write); +boolean vqmsg_push(virtqueue vq, vqmsg m, u64 phys_addr, u32 len, boolean write); void vqmsg_commit_seqno(virtqueue vq, vqmsg m, vqfinish completion, u32 *seqno, boolean kick); #define vqmsg_commit(vq, m, completion) vqmsg_commit_seqno(vq, m, completion, 0, true) void virtqueue_kick(virtqueue vq); diff --git a/src/virtio/virtio_net.c b/src/virtio/virtio_net.c index 38a8fad42..7c609a69f 100644 --- a/src/virtio/virtio_net.c +++ b/src/virtio/virtio_net.c @@ -115,14 +115,16 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) vnet vn = netif->state; virtqueue txq = vn->txq_map[current_cpu()->id]; - vqmsg m = allocate_vqmsg(txq); + vqmsg m = allocate_vqmsg(txq, 2); assert(m != INVALID_ADDRESS); vqmsg_push(txq, m, vn->empty_phys, vn->net_header_len, false); - pbuf_ref(p); - for (struct pbuf * q = p; q != NULL; q = q->next) - vqmsg_push(txq, m, physical_from_virtual(q->payload), q->len, false); + if (!vqmsg_push(txq, m, physical_from_virtual(q->payload), q->len, false)) { + deallocate_vqmsg(txq, m); + return ERR_MEM; + } + pbuf_ref(p); vqmsg_commit(txq, m, closure((heap)vn->txhandlers, tx_complete, p)); @@ -144,20 +146,20 @@ static err_t low_level_output(struct netif *netif, struct pbuf *p) static vqmsg vnet_rxq_push(vnet vn, xpbuf x, int *desc_count) { virtqueue rxq = x->rx->q; - vqmsg m = allocate_vqmsg(rxq); + boolean modern = vtdev_is_modern(vn->dev) || (vn->dev->features & VIRTIO_F_ANY_LAYOUT); + *desc_count = modern ? 1 : 2; + vqmsg m = allocate_vqmsg(rxq, *desc_count); if (m == INVALID_ADDRESS) return m; int rxbuflen = vn->rxbuflen; pbuf_alloced_custom(PBUF_RAW, rxbuflen, PBUF_REF, &x->p, x + 1, rxbuflen); u64 phys = physical_from_virtual(x + 1); - if (vtdev_is_modern(vn->dev) || (vn->dev->features & VIRTIO_F_ANY_LAYOUT)) { + if (modern) { vqmsg_push(rxq, m, phys, rxbuflen, true); - *desc_count = 1; } else { int header_len = vn->net_header_len; vqmsg_push(rxq, m, phys, header_len, true); vqmsg_push(rxq, m, phys + header_len, rxbuflen - header_len, true); - *desc_count = 2; } return m; } @@ -348,7 +350,7 @@ static boolean vnet_ctrl_cmd(vnet vn, u8 class, u8 cmd, void *data, u32 data_len if (command == INVALID_ADDRESS) return false; virtqueue vq = vn->ctl; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 3); if (m == INVALID_ADDRESS) { deallocate(h, command, sizeof(*command)); return false; diff --git a/src/virtio/virtio_rng.c b/src/virtio/virtio_rng.c index a67054554..c887e1c02 100644 --- a/src/virtio/virtio_rng.c +++ b/src/virtio/virtio_rng.c @@ -45,7 +45,7 @@ static void virtio_rng_fill(entropy_buf ebuf) { virtio_rng_debug("%s: ebuf %p\n", func_ss, ebuf); virtqueue vq = virtio_rng.requestq; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); assert(m != INVALID_ADDRESS); ebuf->filling = true; vqmsg_push(vq, m, ebuf->phys, VIRTIO_RNG_BUFSIZE, true); diff --git a/src/virtio/virtio_scsi.c b/src/virtio/virtio_scsi.c index 680bc884f..de2ea8454 100644 --- a/src/virtio/virtio_scsi.c +++ b/src/virtio/virtio_scsi.c @@ -204,7 +204,7 @@ static void virtio_scsi_enqueue_event(virtio_scsi s, virtio_scsi_event e, vqfini c = closure(s->v->virtio_dev.general, virtio_scsi_event_complete, s, e); virtqueue vq = s->eventq; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, physical_from_virtual(e), sizeof(*e), true); vqmsg_commit(vq, m, c); @@ -256,7 +256,7 @@ static void virtio_scsi_enqueue_request(virtio_scsi s, virtio_scsi_request r, vqfinish f = closure(s->v->virtio_dev.general, virtio_scsi_request_complete, c, s, r, r_phys); virtqueue vq = s->requestq; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 3); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, r_phys + offsetof(virtio_scsi_request, req), sizeof(r->req), false); @@ -320,8 +320,12 @@ static void virtio_scsi_io_commit(virtio_scsi s, virtqueue vq, vqmsg msg, boolea virtio_scsi_request r, u64 r_phys, status_handler completion) { heap h = s->v->virtio_dev.general; - if (write) - vqmsg_push(vq, msg, r_phys + offsetof(virtio_scsi_request, resp), sizeof(r->resp), true); + if (write && + !vqmsg_push(vq, msg, r_phys + offsetof(virtio_scsi_request, resp), sizeof(r->resp), true)) { + deallocate_vqmsg(vq, msg); + apply(completion, timm_oom); + return; + } vsr_complete c = closure(h, virtio_scsi_io_done, completion); assert(c != INVALID_ADDRESS); vqfinish f = closure(h, virtio_scsi_request_complete, c, s, r, r_phys); @@ -349,7 +353,7 @@ static void virtio_scsi_io_sg(virtio_scsi_disk d, boolean write, sg_list sg, ran write ? SCSI_CMD_WRITE_16 : SCSI_CMD_READ_16, &r_phys); cdb = (struct scsi_cdb_readwrite_16 *)r->req.cdb; cdb->addr = htobe64(blocks.start); - msg = allocate_vqmsg(vq); + msg = allocate_vqmsg(vq, 3); assert(msg != INVALID_ADDRESS); vqmsg_push(vq, msg, r_phys + offsetof(virtio_scsi_request, req), sizeof(r->req), false); if (!write) @@ -364,7 +368,13 @@ static void virtio_scsi_io_sg(virtio_scsi_disk d, boolean write, sg_list sg, ran length = MIN(range_span(blocks) * d->block_size, length); if (d->max_xfer_len) length = MIN((d->max_xfer_len - req_blocks) * d->block_size, length); - vqmsg_push(vq, msg, physical_from_virtual(sgb->buf + sgb->offset), length, !write); + if (!vqmsg_push(vq, msg, physical_from_virtual(sgb->buf + sgb->offset), length, !write)) { + deallocate_vqmsg(vq, msg); + if (m) + sh = apply_merge(m); + apply(sh, timm_oom); + return; + } sg_consume(sg, length); desc_blocks = length / d->block_size; req_blocks += desc_blocks; diff --git a/src/virtio/virtio_socket.c b/src/virtio/virtio_socket.c index a2b7a0ec5..72e13706b 100644 --- a/src/virtio/virtio_socket.c +++ b/src/virtio/virtio_socket.c @@ -214,7 +214,7 @@ static boolean virtio_sock_tx_hdr(virtio_sock vs, virtio_sock_connection conn, u hdr->fwd_cnt = conn->fwd_cnt; virtio_sock_debug("tx op %d, fwd_cnt %d", op, hdr->fwd_cnt); virtqueue vq = vs->txq; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 1); if (m == INVALID_ADDRESS) { dealloc_unmap(vs->backed, txbuf, phys, sizeof(*txbuf) + sizeof(*hdr)); return false; @@ -357,7 +357,7 @@ static boolean virtio_sock_rxq_submit(virtio_sock vs) virtio_sock_rxbuf rxbuf = alloc_map(vs->backed, VIRTIO_SOCK_RXBUF_SIZE, &phys); if (rxbuf == INVALID_ADDRESS) break; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 2); if (m == INVALID_ADDRESS) { dealloc_unmap(vs->backed, rxbuf, phys, VIRTIO_SOCK_RXBUF_SIZE); break; @@ -461,7 +461,7 @@ boolean virtio_sock_tx(vsock_connection conn, void *data) hdr->fwd_cnt = c->fwd_cnt; virtio_sock_debug("tx data, fwd_cnt %d", hdr->fwd_cnt); virtqueue vq = vs->txq; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 2); if (m == INVALID_ADDRESS) return false; c->tx_cnt += hdr->len; diff --git a/src/virtio/virtio_storage.c b/src/virtio/virtio_storage.c index 2baf0cca0..1677101ac 100644 --- a/src/virtio/virtio_storage.c +++ b/src/virtio/virtio_storage.c @@ -156,7 +156,7 @@ static inline void storage_rw_internal(storage st, boolean write, void * buf, return; } virtqueue vq = st->command; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 3); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, req_phys, VIRTIO_BLK_REQ_HEADER_SIZE, false); vqmsg_push(vq, m, physical_from_virtual(buf), nsectors * st->block_size, !write); @@ -173,7 +173,12 @@ static inline void storage_rw_internal(storage st, boolean write, void * buf, static void virtio_storage_io_commit(storage st, virtqueue vq, vqmsg msg, virtio_blk_req req, u64 req_phys, status_handler completion) { - vqmsg_push(vq, msg, req_phys + VIRTIO_BLK_REQ_HEADER_SIZE, VIRTIO_BLK_REQ_STATUS_SIZE, true); + if (!vqmsg_push(vq, msg, req_phys + VIRTIO_BLK_REQ_HEADER_SIZE, VIRTIO_BLK_REQ_STATUS_SIZE, + true)) { + deallocate_vqmsg(vq, msg); + apply(completion, timm_oom); + return; + } vqfinish c = closure(st->v->general, complete, st, completion, req, req_phys); assert(c != INVALID_ADDRESS); vqmsg_commit(vq, msg, c); @@ -198,7 +203,7 @@ static void virtio_storage_io_sg(storage st, boolean write, sg_list sg, range bl apply(sh, timm_oom); return; } - msg = allocate_vqmsg(vq); + msg = allocate_vqmsg(vq, 3); assert(msg != INVALID_ADDRESS); vqmsg_push(vq, msg, req_phys, VIRTIO_BLK_REQ_HEADER_SIZE, false); desc_count = 0; @@ -207,7 +212,13 @@ static void virtio_storage_io_sg(storage st, boolean write, sg_list sg, range bl u64 length = sg_buf_len(sgb); assert((length & (st->block_size - 1)) == 0); length = MIN(range_span(blocks) * st->block_size, length); - vqmsg_push(vq, msg, physical_from_virtual(sgb->buf + sgb->offset), length, !write); + if (!vqmsg_push(vq, msg, physical_from_virtual(sgb->buf + sgb->offset), length, !write)) { + deallocate_vqmsg(vq, msg); + if (m) + sh = apply_merge(m); + apply(sh, timm_oom); + return; + } sg_consume(sg, length); blocks.start += length / st->block_size; if (++desc_count == st->seg_max) { @@ -236,7 +247,7 @@ static void storage_flush(storage st, status_handler s) return; } virtqueue vq = st->command; - vqmsg m = allocate_vqmsg(vq); + vqmsg m = allocate_vqmsg(vq, 2); assert(m != INVALID_ADDRESS); vqmsg_push(vq, m, req_phys, VIRTIO_BLK_REQ_HEADER_SIZE, false); vqmsg_push(vq, m, req_phys + VIRTIO_BLK_REQ_HEADER_SIZE, VIRTIO_BLK_REQ_STATUS_SIZE, true); diff --git a/src/virtio/virtqueue.c b/src/virtio/virtqueue.c index 14238f8e0..4cfc65d50 100644 --- a/src/virtio/virtqueue.c +++ b/src/virtio/virtqueue.c @@ -109,11 +109,10 @@ typedef struct virtqueue { vqmsg msgs[0]; } *virtqueue; -/* Most uses here are a chain of 3 or less descriptors. */ -#define VQMSG_DEFAULT_SIZE 3 -vqmsg allocate_vqmsg(virtqueue vq) +vqmsg allocate_vqmsg(virtqueue vq, int num_desc) { vqmsg m; + bytes buf_len = sizeof(struct vring_desc) * num_desc; u64 irqflags = spin_lock_irq(&vq->lock); list l = list_get_next(&vq->free_msgs); if (!l) { @@ -122,16 +121,21 @@ vqmsg allocate_vqmsg(virtqueue vq) m = allocate(h, sizeof(struct vqmsg)); if (m == INVALID_ADDRESS) return INVALID_ADDRESS; - m->descv = allocate_buffer(h, sizeof(struct vring_desc) * VQMSG_DEFAULT_SIZE); + m->descv = allocate_buffer(h, buf_len); if (m->descv == INVALID_ADDRESS) { deallocate(h, m, sizeof(struct vqmsg)); return INVALID_ADDRESS; } } else { m = struct_from_list(l, vqmsg, l); + buffer_clear(m->descv); + if ((buffer_space(m->descv) < buf_len) && + (buffer_set_capacity(m->descv, buf_len) != buf_len)) { + spin_unlock_irq(&vq->lock, irqflags); + return INVALID_ADDRESS; + } list_delete(l); spin_unlock_irq(&vq->lock, irqflags); - buffer_clear(m->descv); } list_init(&m->l); m->count = 0; @@ -139,9 +143,10 @@ vqmsg allocate_vqmsg(virtqueue vq) return m; } -void vqmsg_push(virtqueue vq, vqmsg m, u64 phys_addr, u32 len, boolean write) +boolean vqmsg_push(virtqueue vq, vqmsg m, u64 phys_addr, u32 len, boolean write) { - assert(buffer_extend(m->descv, sizeof(struct vring_desc))); + if (!buffer_extend(m->descv, sizeof(struct vring_desc))) + return false; struct vring_desc * d = buffer_ref(m->descv, m->count * sizeof(struct vring_desc)); d->busaddr = phys_addr; d->len = len; @@ -151,6 +156,7 @@ void vqmsg_push(virtqueue vq, vqmsg m, u64 phys_addr, u32 len, boolean write) m->count++; virtqueue_debug_verbose("%s: vq %s, vqmsg %p, phys_addr 0x%lx, len 0x%x, %s, m->count now %d\n", func_ss, vq->name, m, phys_addr, len, write ? "write" : "read", m->count); + return true; } static void virtqueue_fill(virtqueue vq); @@ -173,6 +179,13 @@ void vqmsg_commit_seqno(virtqueue vq, vqmsg m, vqfinish completion, u32 *seqno, spin_unlock_irq(&vq->lock, irqflags); } +void deallocate_vqmsg(virtqueue vq, vqmsg m) +{ + u64 irqflags = spin_lock_irq(&vq->lock); + list_insert_after(&vq->free_msgs, &m->l); + spin_unlock_irq(&vq->lock, irqflags); +} + void virtqueue_kick(virtqueue vq) { spinlock lock = &vq->lock;