Skip to content

Commit 2d1fe63

Browse files
JeynmannZJeynmannZ
committed
UCT/IB: add meta for uct
1 parent 3328c1a commit 2d1fe63

6 files changed

Lines changed: 138 additions & 19 deletions

File tree

‎src/uct/ib/mlx5/ib_mlx5.c‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,13 +696,38 @@ void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq)
696696
txwq->curr = txwq->qstart;
697697
txwq->sw_pi = 0;
698698
txwq->prev_sw_pi = UINT16_MAX;
699+
txwq->ft_ci = UINT16_MAX;
700+
txwq->next_token = 0;
699701
#if UCS_ENABLE_ASSERT
700702
txwq->hw_ci = 0xFFFF;
701703
txwq->flags = 0;
702704
#endif
703705
uct_ib_fence_info_init(&txwq->fi);
704706
}
705707

708+
ucs_status_t uct_ib_mlx5_txwq_tokens_init(uct_ib_mlx5_txwq_t *txwq)
709+
{
710+
size_t num_bb = UCS_PTR_BYTE_DIFF(txwq->qstart, txwq->qend) /
711+
MLX5_SEND_WQE_BB;
712+
713+
ucs_assert(ucs_is_pow2(num_bb));
714+
ucs_assert(num_bb <= UINT16_MAX + 1ul);
715+
716+
txwq->tokens = ucs_calloc(num_bb, sizeof(*txwq->tokens), "mlx5_token");
717+
if (txwq->tokens == NULL) {
718+
return UCS_ERR_NO_MEMORY;
719+
}
720+
721+
txwq->token_mask = num_bb - 1;
722+
return UCS_OK;
723+
}
724+
725+
void uct_ib_mlx5_txwq_tokens_cleanup(uct_ib_mlx5_txwq_t *txwq)
726+
{
727+
ucs_free(txwq->tokens);
728+
txwq->tokens = NULL;
729+
}
730+
706731
void uct_ib_mlx5_init_wq_buf(uct_ib_mlx5_txwq_t *txwq)
707732
{
708733
memset(txwq->qstart, 0, UCS_PTR_BYTE_DIFF(txwq->qstart, txwq->qend));
@@ -729,10 +754,14 @@ void uct_ib_mlx5_txwq_vfs_populate(uct_ib_mlx5_txwq_t *txwq, void *parent_obj)
729754
UCS_VFS_TYPE_U16, "bb_max");
730755
ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->sig_pi,
731756
UCS_VFS_TYPE_U16, "sig_pi");
757+
ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->ft_ci,
758+
UCS_VFS_TYPE_U16, "ft_ci");
732759
#if UCS_ENABLE_ASSERT
733760
ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive, &txwq->hw_ci,
734761
UCS_VFS_TYPE_U16, "hw_ci");
735762
#endif
763+
ucs_vfs_obj_add_ro_file(parent_obj, ucs_vfs_show_primitive,
764+
&txwq->next_token, UCS_VFS_TYPE_U32, "next_token");
736765
}
737766

738767
ucs_status_t

‎src/uct/ib/mlx5/ib_mlx5.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -686,13 +686,17 @@ typedef struct uct_ib_mlx5_txwq {
686686
uct_ib_mlx5_qp_t super;
687687
uint16_t sw_pi; /* PI for next WQE */
688688
uint16_t prev_sw_pi; /* PI where last WQE *started* */
689+
uint16_t ft_ci; /* Last completed/recovered WQE start */
689690
uct_ib_mlx5_mmio_reg_t *reg;
690691
void *curr;
691692
volatile uint32_t *dbrec;
692693
void *qstart;
693694
void *qend;
694695
uint16_t bb_max;
695696
uint16_t sig_pi; /* PI for last signaled WQE */
697+
uint32_t next_token; /* Token assigned to the next packet */
698+
uint32_t *tokens; /* Token indexed by WQEBB */
699+
uint16_t token_mask; /* Mask for token array */
696700
#if UCS_ENABLE_ASSERT
697701
uint16_t hw_ci; /* First BB index of last completed WQE */
698702
uint8_t flags; /* Debug flags */
@@ -917,6 +921,10 @@ void uct_ib_mlx5_qp_mmio_cleanup(uct_ib_mlx5_qp_t *qp,
917921
*/
918922
void uct_ib_mlx5_txwq_reset(uct_ib_mlx5_txwq_t *txwq);
919923

924+
ucs_status_t uct_ib_mlx5_txwq_tokens_init(uct_ib_mlx5_txwq_t *txwq);
925+
926+
void uct_ib_mlx5_txwq_tokens_cleanup(uct_ib_mlx5_txwq_t *txwq);
927+
920928
void uct_ib_mlx5_init_wq_buf(uct_ib_mlx5_txwq_t *txwq);
921929

922930
/**

‎src/uct/ib/mlx5/rc/rc_mlx5.inl‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -452,16 +452,42 @@ uct_rc_mlx5_ep_fm_cq_update(uct_rc_mlx5_iface_common_t *iface,
452452
return fm_ce_se;
453453
}
454454

455+
static UCS_F_ALWAYS_INLINE void
456+
uct_rc_mlx5_txwq_record_token(uct_rc_mlx5_iface_common_t *iface,
457+
uct_ib_mlx5_txwq_t *txwq, uint16_t wqe_pi,
458+
uint8_t opcode, size_t message_length)
459+
{
460+
size_t mtu;
461+
uint32_t num_packets;
462+
463+
ucs_assert(txwq->tokens != NULL);
464+
465+
txwq->tokens[wqe_pi & txwq->token_mask] = txwq->next_token;
466+
if (opcode == MLX5_OPCODE_NOP) {
467+
num_packets = 0;
468+
} else if ((opcode == MLX5_OPCODE_SEND) ||
469+
(opcode == MLX5_OPCODE_SEND_IMM) ||
470+
(opcode == MLX5_OPCODE_RDMA_WRITE)) {
471+
mtu = uct_ib_mtu_value(iface->super.super.config.path_mtu);
472+
num_packets = ucs_max(1ul, ucs_div_round_up(message_length, mtu));
473+
} else {
474+
num_packets = 1;
475+
}
476+
477+
txwq->next_token = (txwq->next_token + num_packets) & UCS_MASK(24);
478+
}
479+
455480
static UCS_F_ALWAYS_INLINE void
456481
uct_rc_mlx5_common_post_send(uct_rc_mlx5_iface_common_t *iface, int qp_type,
457482
uct_rc_txqp_t *txqp, uct_ib_mlx5_txwq_t *txwq,
458483
uint8_t opcode, uint8_t opmod, uint8_t fm_ce_se,
459484
uint16_t dci_channel, size_t wqe_size,
460-
uint32_t imm, int max_log_sge,
485+
size_t message_length, uint32_t imm,
486+
int max_log_sge,
461487
uct_ib_log_sge_t *log_sge)
462488
{
463489
struct mlx5_wqe_ctrl_seg *ctrl;
464-
uint16_t res_count;
490+
uint16_t res_count, wqe_pi;
465491

466492
if (opcode != MLX5_OPCODE_NOP) {
467493
/* If FAILED, allow only NOP sends to be posted (used by endpoint
@@ -492,7 +518,14 @@ uct_rc_mlx5_common_post_send(uct_rc_mlx5_iface_common_t *iface, int qp_type,
492518
((opcode == MLX5_OPCODE_SEND) || (opcode == MLX5_OPCODE_SEND_IMM)) ?
493519
uct_rc_mlx5_common_packet_dump : NULL);
494520

521+
wqe_pi = txwq->sw_pi;
495522
res_count = uct_ib_mlx5_post_send(txwq, ctrl, wqe_size, 1);
523+
524+
if (qp_type == IBV_QPT_RC) {
525+
uct_rc_mlx5_txwq_record_token(iface, txwq, wqe_pi, opcode,
526+
message_length);
527+
}
528+
496529
if (fm_ce_se & MLX5_WQE_CTRL_CQ_UPDATE) {
497530
txwq->sig_pi = txwq->prev_sw_pi;
498531
}
@@ -530,8 +563,8 @@ static UCS_F_ALWAYS_INLINE void uct_rc_mlx5_txqp_inline_iov_post(
530563
uct_rc_mlx5_am_hdr_fill(rch, am_id);
531564
uct_ib_mlx5_inline_iov_copy(rch + 1, iov, iovcnt, iov_length, txwq);
532565
uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, MLX5_OPCODE_SEND,
533-
0, fm_ce_se, dci_channel, wqe_size, 0,
534-
INT_MAX, NULL);
566+
0, fm_ce_se, dci_channel, wqe_size,
567+
sizeof(*rch) + iov_length, 0, INT_MAX, NULL);
535568
}
536569

537570
/*
@@ -564,7 +597,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
564597
struct mlx5_wqe_inl_data_seg *inl;
565598
uct_rc_mlx5_am_short_hdr_t *am;
566599
uct_rc_mlx5_hdr_t *rc_hdr;
567-
size_t wqe_size, ctrl_av_size;
600+
size_t wqe_size, ctrl_av_size, message_length;
568601
void *next_seg;
569602

570603
ctrl = txwq->curr;
@@ -584,6 +617,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
584617
am->am_hdr = am_hdr;
585618
uct_rc_mlx5_am_hdr_fill(&am->rc_hdr, am_id);
586619
uct_ib_mlx5_inline_copy(am + 1, buffer, length, txwq);
620+
message_length = length + sizeof(*am);
587621
fm_ce_se |= uct_rc_iface_tx_moderation(&iface->super, txqp, MLX5_WQE_CTRL_CQ_UPDATE);
588622
break;
589623

@@ -594,6 +628,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
594628
inl->byte_count = htonl(sizeof(*rc_hdr) | MLX5_INLINE_SEG);
595629
rc_hdr = (void*)(inl + 1);
596630
uct_rc_mlx5_am_hdr_fill(rc_hdr, am_id);
631+
message_length = sizeof(*rc_hdr);
597632
fm_ce_se |= uct_rc_iface_tx_moderation(&iface->super, txqp, MLX5_WQE_CTRL_CQ_UPDATE);
598633
break;
599634

@@ -609,6 +644,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
609644
inl = uct_ib_mlx5_txwq_wrap_none(txwq, raddr + 1);
610645
inl->byte_count = htonl(length | MLX5_INLINE_SEG);
611646
uct_ib_mlx5_inline_copy(inl + 1, buffer, length, txwq);
647+
message_length = length;
612648
fm_ce_se |= uct_rc_iface_tx_moderation(&iface->super, txqp, MLX5_WQE_CTRL_CQ_UPDATE);
613649
break;
614650

@@ -617,6 +653,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
617653
wqe_size = sizeof(*ctrl) + av_size;
618654
inl = next_seg;
619655
inl->byte_count = htonl(MLX5_INLINE_SEG);
656+
message_length = 0;
620657
fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE | MLX5_WQE_CTRL_FENCE;
621658
break;
622659

@@ -625,7 +662,7 @@ uct_rc_mlx5_txqp_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
625662
}
626663

627664
uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0, fm_ce_se,
628-
dci_channel, wqe_size, imm_val_be,
665+
dci_channel, wqe_size, message_length, imm_val_be,
629666
max_log_sge, NULL);
630667
}
631668

@@ -667,7 +704,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
667704
struct uct_ib_mlx5_atomic_masked_fadd32_seg *masked_fadd32;
668705
struct uct_ib_mlx5_atomic_masked_cswap64_seg *masked_cswap64;
669706
struct uct_ib_mlx5_atomic_masked_fadd64_seg *masked_fadd64;
670-
size_t wqe_size, ctrl_av_size;
707+
size_t wqe_size, ctrl_av_size, message_length = 0;
671708
uint8_t opmod;
672709
void *next_seg;
673710

@@ -692,6 +729,7 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
692729

693730
wqe_size = ctrl_av_size + sizeof(struct mlx5_wqe_data_seg);
694731
uct_ib_mlx5_set_data_seg(next_seg, buffer, length, *lkey_p);
732+
message_length = length;
695733
break;
696734

697735
case MLX5_OPCODE_RDMA_READ:
@@ -713,6 +751,9 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
713751
wqe_size = ctrl_av_size + sizeof(*raddr) + sizeof(*dptr);
714752
uct_ib_mlx5_set_data_seg(dptr, buffer, length, *lkey_p);
715753
}
754+
if (opcode_flags == MLX5_OPCODE_RDMA_WRITE) {
755+
message_length = length;
756+
}
716757
break;
717758

718759
case MLX5_OPCODE_ATOMIC_FA:
@@ -808,8 +849,9 @@ uct_rc_mlx5_txqp_dptr_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
808849

809850
uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq,
810851
(opcode_flags & UCT_RC_MLX5_OPCODE_MASK), opmod,
811-
fm_ce_se, dci_channel, wqe_size, imm_val_be,
812-
max_log_sge, log_sge);
852+
fm_ce_se, dci_channel, wqe_size,
853+
message_length, imm_val_be, max_log_sge,
854+
log_sge);
813855
}
814856

815857
static UCS_F_ALWAYS_INLINE
@@ -830,6 +872,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
830872
struct mlx5_wqe_inl_data_seg *inl;
831873
uct_rc_mlx5_hdr_t *rch;
832874
unsigned wqe_size, inl_seg_size, ctrl_av_size;
875+
size_t iov_length, message_length = 0;
833876
void *next_seg;
834877
uint8_t opmod;
835878
#if HAVE_MLX5_MMO
@@ -844,6 +887,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
844887

845888
ctrl = txwq->curr;
846889
ctrl_av_size = sizeof(*ctrl) + av_size;
890+
iov_length = uct_iov_total_length(iov, iovcnt);
847891
next_seg = UCS_PTR_BYTE_OFFSET(ctrl, ctrl_av_size);
848892
next_seg = uct_ib_mlx5_txwq_wrap_exact(txwq, next_seg);
849893

@@ -852,7 +896,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
852896
inl_seg_size = ucs_align_up_pow2(sizeof(*inl) + sizeof(*rch) + am_hdr_len,
853897
UCT_IB_MLX5_WQE_SEG_SIZE);
854898

855-
ucs_assert(uct_iov_total_length(iov, iovcnt) + sizeof(*rch) + am_hdr_len <=
899+
ucs_assert(iov_length + sizeof(*rch) + am_hdr_len <=
856900
iface->super.super.config.seg_size);
857901

858902
/* Inline segment with AM ID and header */
@@ -868,6 +912,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
868912
wqe_size = ctrl_av_size + inl_seg_size +
869913
uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt);
870914
opmod = 0;
915+
message_length = iov_length + sizeof(*rch) + am_hdr_len;
871916

872917
ucs_assert(wqe_size <= UCT_IB_MLX5_MAX_SEND_WQE_SIZE);
873918
break;
@@ -883,6 +928,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
883928
wqe_size = ctrl_av_size + inl_seg_size +
884929
uct_ib_mlx5_set_data_seg_iov(txwq, dptr, iov, iovcnt);
885930
opmod = 0;
931+
message_length = iov_length + sizeof(struct ibv_tmh);
886932

887933
uct_rc_mlx5_fill_tmh((struct ibv_tmh*)(inl + 1), tag, app_ctx,
888934
IBV_TMH_EAGER);
@@ -895,7 +941,7 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
895941
/* Fall through */
896942
case MLX5_OPCODE_RDMA_WRITE:
897943
/* Set RDMA segment */
898-
ucs_assert(uct_iov_total_length(iov, iovcnt) <= UCT_IB_MAX_MESSAGE_SIZE);
944+
ucs_assert(iov_length <= UCT_IB_MAX_MESSAGE_SIZE);
899945

900946
raddr = next_seg;
901947
uct_ib_mlx5_ep_set_rdma_seg(raddr, remote_addr, rkey);
@@ -905,6 +951,9 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
905951
uct_ib_mlx5_set_data_seg_iov(txwq, (void*)(raddr + 1),
906952
iov, iovcnt);
907953
opmod = 0;
954+
if (opcode_flags == MLX5_OPCODE_RDMA_WRITE) {
955+
message_length = iov_length;
956+
}
908957
break;
909958

910959
#if HAVE_MLX5_MMO
@@ -948,8 +997,8 @@ void uct_rc_mlx5_txqp_dptr_post_iov(uct_rc_mlx5_iface_common_t *iface, int qp_ty
948997

949998
uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq,
950999
opcode_flags & UCT_RC_MLX5_OPCODE_MASK, opmod,
951-
fm_ce_se, dci_channel, wqe_size, ib_imm_be,
952-
max_log_sge, NULL);
1000+
fm_ce_se, dci_channel, wqe_size,
1001+
message_length, ib_imm_be, max_log_sge, NULL);
9531002
}
9541003

9551004
/*
@@ -1097,8 +1146,9 @@ uct_rc_mlx5_txqp_tag_inline_post(uct_rc_mlx5_iface_common_t *iface, int qp_type,
10971146
fm_ce_se |= uct_rc_iface_tx_moderation(&iface->super, txqp, MLX5_WQE_CTRL_CQ_UPDATE);
10981147

10991148
uct_rc_mlx5_common_post_send(iface, qp_type, txqp, txwq, opcode, 0,
1100-
fm_ce_se, dci_channel, wqe_size, imm_val_be,
1101-
INT_MAX, NULL);
1149+
fm_ce_se, dci_channel, wqe_size,
1150+
ntohl(inl->byte_count) & ~MLX5_INLINE_SEG,
1151+
imm_val_be, INT_MAX, NULL);
11021152
}
11031153

11041154
static UCS_F_ALWAYS_INLINE void

0 commit comments

Comments
 (0)