From d8596eff4eb46d1db1b6066e3bbbd03976f49e58 Mon Sep 17 00:00:00 2001 From: Xinhao Liu Date: Mon, 7 Mar 2022 18:49:35 +0800 Subject: libhns: Add support for creating extended CQ The driver supports ibv_create_cq_ex() to create extended CQ. But the driver does not yet support the extended attributes specified by attr->com_mask and attr->wc_flas. Signed-off-by: Xinhao Liu Signed-off-by: Wenpeng Liang --- providers/hns/hns_roce_u.c | 1 + providers/hns/hns_roce_u.h | 6 ++- providers/hns/hns_roce_u_abi.h | 5 +++ providers/hns/hns_roce_u_hw_v1.c | 20 +++++----- providers/hns/hns_roce_u_hw_v2.c | 16 ++++---- providers/hns/hns_roce_u_verbs.c | 63 ++++++++++++++++++++++---------- 6 files changed, 72 insertions(+), 39 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index 6eac4ff0..f8a647b8 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -66,6 +66,7 @@ static const struct verbs_context_ops hns_common_ops = { .bind_mw = hns_roce_u_bind_mw, .cq_event = hns_roce_u_cq_event, .create_cq = hns_roce_u_create_cq, + .create_cq_ex = hns_roce_u_create_cq_ex, .create_qp = hns_roce_u_create_qp, .create_qp_ex = hns_roce_u_create_qp_ex, .dealloc_mw = hns_roce_u_dealloc_mw, diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index 2b4ba181..505e7498 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -236,7 +236,7 @@ struct hns_roce_pd { }; struct hns_roce_cq { - struct ibv_cq ibv_cq; + struct verbs_cq verbs_cq; struct hns_roce_buf buf; pthread_spinlock_t lock; unsigned int cqn; @@ -406,7 +406,7 @@ static inline struct hns_roce_pd *to_hr_pd(struct ibv_pd *ibv_pd) static inline struct hns_roce_cq *to_hr_cq(struct ibv_cq *ibv_cq) { - return container_of(ibv_cq, struct hns_roce_cq, ibv_cq); + return container_of(ibv_cq, struct hns_roce_cq, verbs_cq.cq); } static inline struct hns_roce_srq *to_hr_srq(struct ibv_srq *ibv_srq) @@ -447,6 +447,8 @@ int hns_roce_u_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, int comp_vector); +struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr); int hns_roce_u_modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); int hns_roce_u_destroy_cq(struct ibv_cq *cq); diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h index e56f9d35..333f977e 100644 --- a/providers/hns/hns_roce_u_abi.h +++ b/providers/hns/hns_roce_u_abi.h @@ -39,8 +39,13 @@ DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, hns_roce_ib_alloc_pd_resp); + DECLARE_DRV_CMD(hns_roce_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp); + +DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, + hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp); + DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty, hns_roce_ib_alloc_ucontext_resp); diff --git a/providers/hns/hns_roce_u_hw_v1.c b/providers/hns/hns_roce_u_hw_v1.c index 28ad482c..d47cba0c 100644 --- a/providers/hns/hns_roce_u_hw_v1.c +++ b/providers/hns/hns_roce_u_hw_v1.c @@ -161,10 +161,10 @@ static struct hns_roce_cqe *get_cqe(struct hns_roce_cq *cq, int entry) static void *get_sw_cqe(struct hns_roce_cq *cq, int n) { - struct hns_roce_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe); + struct hns_roce_cqe *cqe = get_cqe(cq, n & cq->verbs_cq.cq.cqe); return (!!(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S)) ^ - !!(n & (cq->ibv_cq.cqe + 1))) ? cqe : NULL; + !!(n & (cq->verbs_cq.cq.cqe + 1))) ? cqe : NULL; } static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *cq) @@ -210,7 +210,7 @@ static int hns_roce_wq_overflow(struct hns_roce_wq *wq, int nreq, cur = wq->head - wq->tail; pthread_spin_unlock(&cq->lock); - verbs_err(verbs_get_ctx(cq->ibv_cq.context), + verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context), "wq:(head = %d, tail = %d, max_post = %d), nreq = 0x%x\n", wq->head, wq->tail, wq->max_post, nreq); @@ -274,10 +274,10 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *cq, if (!*cur_qp || (local_qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->verbs_qp.qp.qp_num) { - *cur_qp = hns_roce_find_qp(to_hr_ctx(cq->ibv_cq.context), + *cur_qp = hns_roce_find_qp(to_hr_ctx(cq->verbs_cq.cq.context), qpn & 0xffffff); if (!*cur_qp) { - verbs_err(verbs_get_ctx(cq->ibv_cq.context), + verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context), PFX "can't find qp!\n"); return CQ_POLL_ERR; } @@ -317,7 +317,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *cq, if (roce_get_field(cqe->cqe_byte_4, CQE_BYTE_4_STATUS_OF_THE_OPERATION_M, CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) != HNS_ROCE_CQE_SUCCESS) { - verbs_err(verbs_get_ctx(cq->ibv_cq.context), + verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context), PFX "error cqe!\n"); hns_roce_handle_error_cqe(cqe, wc); return CQ_OK; @@ -599,21 +599,21 @@ static void __hns_roce_v1_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, uint32_t prod_index; uint8_t owner_bit = 0; struct hns_roce_cqe *cqe, *dest; - struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context); + struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context); for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index) - if (prod_index == cq->cons_index + cq->ibv_cq.cqe) + if (prod_index == cq->cons_index + cq->verbs_cq.cq.cqe) break; while ((int) --prod_index - (int) cq->cons_index >= 0) { - cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); + cqe = get_cqe(cq, prod_index & cq->verbs_cq.cq.cqe); if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, CQE_BYTE_16_LOCAL_QPN_S) & 0xffffff) == qpn) { ++nfreed; } else if (nfreed) { dest = get_cqe(cq, - (prod_index + nfreed) & cq->ibv_cq.cqe); + (prod_index + nfreed) & cq->verbs_cq.cq.cqe); owner_bit = roce_get_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S); memcpy(dest, cqe, sizeof(*cqe)); diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index bfd98760..07f3596d 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -189,10 +189,10 @@ static struct hns_roce_v2_cqe *get_cqe_v2(struct hns_roce_cq *cq, int entry) static void *get_sw_cqe_v2(struct hns_roce_cq *cq, int n) { - struct hns_roce_v2_cqe *cqe = get_cqe_v2(cq, n & cq->ibv_cq.cqe); + struct hns_roce_v2_cqe *cqe = get_cqe_v2(cq, n & cq->verbs_cq.cq.cqe); - return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & (cq->ibv_cq.cqe + 1))) ? - cqe : NULL; + return (hr_reg_read(cqe, CQE_OWNER) ^ + !!(n & (cq->verbs_cq.cq.cqe + 1))) ? cqe : NULL; } static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *cq) @@ -556,7 +556,7 @@ static void parse_cqe_for_req(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, static int hns_roce_v2_poll_one(struct hns_roce_cq *cq, struct hns_roce_qp **cur_qp, struct ibv_wc *wc) { - struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context); + struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context); struct hns_roce_srq *srq = NULL; struct hns_roce_v2_cqe *cqe; uint8_t opcode; @@ -1356,15 +1356,15 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, uint16_t wqe_index; uint32_t prod_index; struct hns_roce_v2_cqe *cqe, *dest; - struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context); + struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context); for (prod_index = cq->cons_index; get_sw_cqe_v2(cq, prod_index); ++prod_index) - if (prod_index > cq->cons_index + cq->ibv_cq.cqe) + if (prod_index > cq->cons_index + cq->verbs_cq.cq.cqe) break; while ((int) --prod_index - (int) cq->cons_index >= 0) { - cqe = get_cqe_v2(cq, prod_index & cq->ibv_cq.cqe); + cqe = get_cqe_v2(cq, prod_index & cq->verbs_cq.cq.cqe); if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) { is_recv_cqe = hr_reg_read(cqe, CQE_S_R); @@ -1375,7 +1375,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn, ++nfreed; } else if (nfreed) { dest = get_cqe_v2(cq, - (prod_index + nfreed) & cq->ibv_cq.cqe); + (prod_index + nfreed) & cq->verbs_cq.cq.cqe); owner_bit = hr_reg_read(dest, CQE_OWNER); memcpy(dest, cqe, cq->cqe_size); hr_reg_write_bool(dest, CQE_OWNER, owner_bit); diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 3cc9e0c2..a993c39a 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -276,12 +276,17 @@ int hns_roce_u_dealloc_mw(struct ibv_mw *mw) return 0; } -static int hns_roce_verify_cq(int *cqe, struct hns_roce_context *context) +static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr, + struct hns_roce_context *context) { - if (*cqe < 1 || *cqe > context->max_cqe) + if (!attr->cqe || attr->cqe > context->max_cqe) return -EINVAL; - *cqe = max((uint64_t)HNS_ROCE_MIN_CQE_NUM, roundup_pow_of_two(*cqe)); + if (attr->comp_mask || attr->wc_flags) + return -EOPNOTSUPP; + + attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM, + roundup_pow_of_two(attr->cqe)); return 0; } @@ -297,25 +302,25 @@ static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq) } static int exec_cq_create_cmd(struct ibv_context *context, - struct hns_roce_cq *cq, int cqe, - struct ibv_comp_channel *channel, int comp_vector) + struct hns_roce_cq *cq, + struct ibv_cq_init_attr_ex *attr) { + struct hns_roce_create_cq_ex_resp resp_ex = {}; struct hns_roce_ib_create_cq_resp *resp_drv; - struct hns_roce_create_cq_resp resp = {}; + struct hns_roce_create_cq_ex cmd_ex = {}; struct hns_roce_ib_create_cq *cmd_drv; - struct hns_roce_create_cq cmd = {}; int ret; - cmd_drv = &cmd.drv_payload; - resp_drv = &resp.drv_payload; + cmd_drv = &cmd_ex.drv_payload; + resp_drv = &resp_ex.drv_payload; cmd_drv->buf_addr = (uintptr_t)cq->buf.buf; cmd_drv->db_addr = (uintptr_t)cq->db; cmd_drv->cqe_size = (uintptr_t)cq->cqe_size; - ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector, - &cq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd), - &resp.ibv_resp, sizeof(resp)); + ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex), 0); if (ret) return ret; @@ -325,16 +330,15 @@ static int exec_cq_create_cmd(struct ibv_context *context, return 0; } -struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, - struct ibv_comp_channel *channel, - int comp_vector) +static struct ibv_cq_ex *create_cq(struct ibv_context *context, + struct ibv_cq_init_attr_ex *attr) { struct hns_roce_device *hr_dev = to_hr_dev(context->device); struct hns_roce_context *hr_ctx = to_hr_ctx(context); struct hns_roce_cq *cq; int ret; - ret = hns_roce_verify_cq(&cqe, hr_ctx); + ret = verify_cq_create_attr(attr, hr_ctx); if (ret) goto err; @@ -348,7 +352,7 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, if (ret) goto err_lock; - cq->cq_depth = cqe; + cq->cq_depth = attr->cqe; cq->cqe_size = hr_ctx->cqe_size; ret = hns_roce_alloc_cq_buf(cq); @@ -363,13 +367,13 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, *cq->db = 0; - ret = exec_cq_create_cmd(context, cq, cqe, channel, comp_vector); + ret = exec_cq_create_cmd(context, cq, attr); if (ret) goto err_cmd; cq->arm_sn = 1; - return &cq->ibv_cq; + return &cq->verbs_cq.cq_ex; err_cmd: if (hr_dev->hw_version != HNS_ROCE_HW_VER1) @@ -387,6 +391,27 @@ err: return NULL; } +struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector) +{ + struct ibv_cq_ex *cq; + struct ibv_cq_init_attr_ex attr = { + .cqe = cqe, + .channel = channel, + .comp_vector = comp_vector, + }; + + cq = create_cq(context, &attr); + return cq ? ibv_cq_ex_to_cq(cq) : NULL; +} + +struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context, + struct ibv_cq_init_attr_ex *attr) +{ + return create_cq(context, attr); +} + void hns_roce_u_cq_event(struct ibv_cq *cq) { to_hr_cq(cq)->arm_sn++; -- 2.30.0