packages/rdma-core
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support CQ_EX for hns

Browse Source 
Add support for hns CQ_EX and all its dependent patches are also added.

Signed-off-by: zhengfeng luo <luozhengfeng@h-partners.com>
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
This commit is contained in:
zhengfeng luo 2022-08-29 20:33:05 +08:00 committed by Chengchang Tang
parent 123f94ea3f
commit 910edf6453
7 changed files with 2322 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

346
0040-libhns-Add-support-for-creating-extended-CQ.patch Normal file
View File

@ -0,0 +1,346 @@
From d8596eff4eb46d1db1b6066e3bbbd03976f49e58 Mon Sep 17 00:00:00 2001
From: Xinhao Liu <liuxinhao5@hisilicon.com>
Date: Mon, 7 Mar 2022 18:49:35 +0800
Subject: libhns: Add support for creating extended CQ
The driver supports ibv_create_cq_ex() to create extended CQ. But the
driver does not yet support the extended attributes specified by
attr->com_mask and attr->wc_flas.
Signed-off-by: Xinhao Liu <liuxinhao5@hisilicon.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u.c | 1 +
providers/hns/hns_roce_u.h | 6 ++-
providers/hns/hns_roce_u_abi.h | 5 +++
providers/hns/hns_roce_u_hw_v1.c | 20 +++++-----
providers/hns/hns_roce_u_hw_v2.c | 16 ++++----
providers/hns/hns_roce_u_verbs.c | 63 ++++++++++++++++++++++----------
6 files changed, 72 insertions(+), 39 deletions(-)
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 6eac4ff0..f8a647b8 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -66,6 +66,7 @@ static const struct verbs_context_ops hns_common_ops = {
.bind_mw = hns_roce_u_bind_mw,
.cq_event = hns_roce_u_cq_event,
.create_cq = hns_roce_u_create_cq,
+ .create_cq_ex = hns_roce_u_create_cq_ex,
.create_qp = hns_roce_u_create_qp,
.create_qp_ex = hns_roce_u_create_qp_ex,
.dealloc_mw = hns_roce_u_dealloc_mw,
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 2b4ba181..505e7498 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -236,7 +236,7 @@ struct hns_roce_pd {
};
struct hns_roce_cq {
- struct ibv_cq ibv_cq;
+ struct verbs_cq verbs_cq;
struct hns_roce_buf buf;
pthread_spinlock_t lock;
unsigned int cqn;
@@ -406,7 +406,7 @@ static inline struct hns_roce_pd *to_hr_pd(struct ibv_pd *ibv_pd)
static inline struct hns_roce_cq *to_hr_cq(struct ibv_cq *ibv_cq)
{
- return container_of(ibv_cq, struct hns_roce_cq, ibv_cq);
+ return container_of(ibv_cq, struct hns_roce_cq, verbs_cq.cq);
}
static inline struct hns_roce_srq *to_hr_srq(struct ibv_srq *ibv_srq)
@@ -447,6 +447,8 @@ int hns_roce_u_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
struct ibv_comp_channel *channel,
int comp_vector);
+struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *cq_attr);
int hns_roce_u_modify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr);
int hns_roce_u_destroy_cq(struct ibv_cq *cq);
diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
index e56f9d35..333f977e 100644
--- a/providers/hns/hns_roce_u_abi.h
+++ b/providers/hns/hns_roce_u_abi.h
@@ -39,8 +39,13 @@
DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
empty, hns_roce_ib_alloc_pd_resp);
+
DECLARE_DRV_CMD(hns_roce_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp);
+
+DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
+ hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp);
+
DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
empty, hns_roce_ib_alloc_ucontext_resp);
diff --git a/providers/hns/hns_roce_u_hw_v1.c b/providers/hns/hns_roce_u_hw_v1.c
index 28ad482c..d47cba0c 100644
--- a/providers/hns/hns_roce_u_hw_v1.c
+++ b/providers/hns/hns_roce_u_hw_v1.c
@@ -161,10 +161,10 @@ static struct hns_roce_cqe *get_cqe(struct hns_roce_cq *cq, int entry)
static void *get_sw_cqe(struct hns_roce_cq *cq, int n)
{
- struct hns_roce_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
+ struct hns_roce_cqe *cqe = get_cqe(cq, n & cq->verbs_cq.cq.cqe);
return (!!(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S)) ^
- !!(n & (cq->ibv_cq.cqe + 1))) ? cqe : NULL;
+ !!(n & (cq->verbs_cq.cq.cqe + 1))) ? cqe : NULL;
}
static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *cq)
@@ -210,7 +210,7 @@ static int hns_roce_wq_overflow(struct hns_roce_wq *wq, int nreq,
cur = wq->head - wq->tail;
pthread_spin_unlock(&cq->lock);
- verbs_err(verbs_get_ctx(cq->ibv_cq.context),
+ verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context),
"wq:(head = %d, tail = %d, max_post = %d), nreq = 0x%x\n",
wq->head, wq->tail, wq->max_post, nreq);
@@ -274,10 +274,10 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *cq,
if (!*cur_qp ||
(local_qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->verbs_qp.qp.qp_num) {
- *cur_qp = hns_roce_find_qp(to_hr_ctx(cq->ibv_cq.context),
+ *cur_qp = hns_roce_find_qp(to_hr_ctx(cq->verbs_cq.cq.context),
qpn & 0xffffff);
if (!*cur_qp) {
- verbs_err(verbs_get_ctx(cq->ibv_cq.context),
+ verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context),
PFX "can't find qp!\n");
return CQ_POLL_ERR;
}
@@ -317,7 +317,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *cq,
if (roce_get_field(cqe->cqe_byte_4,
CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) != HNS_ROCE_CQE_SUCCESS) {
- verbs_err(verbs_get_ctx(cq->ibv_cq.context),
+ verbs_err(verbs_get_ctx(cq->verbs_cq.cq.context),
PFX "error cqe!\n");
hns_roce_handle_error_cqe(cqe, wc);
return CQ_OK;
@@ -599,21 +599,21 @@ static void __hns_roce_v1_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
uint32_t prod_index;
uint8_t owner_bit = 0;
struct hns_roce_cqe *cqe, *dest;
- struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context);
+ struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context);
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index);
++prod_index)
- if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
+ if (prod_index == cq->cons_index + cq->verbs_cq.cq.cqe)
break;
while ((int) --prod_index - (int) cq->cons_index >= 0) {
- cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
+ cqe = get_cqe(cq, prod_index & cq->verbs_cq.cq.cqe);
if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
CQE_BYTE_16_LOCAL_QPN_S) & 0xffffff) == qpn) {
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq,
- (prod_index + nfreed) & cq->ibv_cq.cqe);
+ (prod_index + nfreed) & cq->verbs_cq.cq.cqe);
owner_bit = roce_get_bit(dest->cqe_byte_4,
CQE_BYTE_4_OWNER_S);
memcpy(dest, cqe, sizeof(*cqe));
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index bfd98760..07f3596d 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -189,10 +189,10 @@ static struct hns_roce_v2_cqe *get_cqe_v2(struct hns_roce_cq *cq, int entry)
static void *get_sw_cqe_v2(struct hns_roce_cq *cq, int n)
{
- struct hns_roce_v2_cqe *cqe = get_cqe_v2(cq, n & cq->ibv_cq.cqe);
+ struct hns_roce_v2_cqe *cqe = get_cqe_v2(cq, n & cq->verbs_cq.cq.cqe);
- return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & (cq->ibv_cq.cqe + 1))) ?
- cqe : NULL;
+ return (hr_reg_read(cqe, CQE_OWNER) ^
+ !!(n & (cq->verbs_cq.cq.cqe + 1))) ? cqe : NULL;
}
static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *cq)
@@ -556,7 +556,7 @@ static void parse_cqe_for_req(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
struct hns_roce_qp **cur_qp, struct ibv_wc *wc)
{
- struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context);
+ struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context);
struct hns_roce_srq *srq = NULL;
struct hns_roce_v2_cqe *cqe;
uint8_t opcode;
@@ -1356,15 +1356,15 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
uint16_t wqe_index;
uint32_t prod_index;
struct hns_roce_v2_cqe *cqe, *dest;
- struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context);
+ struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context);
for (prod_index = cq->cons_index; get_sw_cqe_v2(cq, prod_index);
++prod_index)
- if (prod_index > cq->cons_index + cq->ibv_cq.cqe)
+ if (prod_index > cq->cons_index + cq->verbs_cq.cq.cqe)
break;
while ((int) --prod_index - (int) cq->cons_index >= 0) {
- cqe = get_cqe_v2(cq, prod_index & cq->ibv_cq.cqe);
+ cqe = get_cqe_v2(cq, prod_index & cq->verbs_cq.cq.cqe);
if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
is_recv_cqe = hr_reg_read(cqe, CQE_S_R);
@@ -1375,7 +1375,7 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
++nfreed;
} else if (nfreed) {
dest = get_cqe_v2(cq,
- (prod_index + nfreed) & cq->ibv_cq.cqe);
+ (prod_index + nfreed) & cq->verbs_cq.cq.cqe);
owner_bit = hr_reg_read(dest, CQE_OWNER);
memcpy(dest, cqe, cq->cqe_size);
hr_reg_write_bool(dest, CQE_OWNER, owner_bit);
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 3cc9e0c2..a993c39a 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -276,12 +276,17 @@ int hns_roce_u_dealloc_mw(struct ibv_mw *mw)
return 0;
}
-static int hns_roce_verify_cq(int *cqe, struct hns_roce_context *context)
+static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
+ struct hns_roce_context *context)
{
- if (*cqe < 1 || *cqe > context->max_cqe)
+ if (!attr->cqe || attr->cqe > context->max_cqe)
return -EINVAL;
- *cqe = max((uint64_t)HNS_ROCE_MIN_CQE_NUM, roundup_pow_of_two(*cqe));
+ if (attr->comp_mask || attr->wc_flags)
+ return -EOPNOTSUPP;
+
+ attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM,
+ roundup_pow_of_two(attr->cqe));
return 0;
}
@@ -297,25 +302,25 @@ static int hns_roce_alloc_cq_buf(struct hns_roce_cq *cq)
}
static int exec_cq_create_cmd(struct ibv_context *context,
- struct hns_roce_cq *cq, int cqe,
- struct ibv_comp_channel *channel, int comp_vector)
+ struct hns_roce_cq *cq,
+ struct ibv_cq_init_attr_ex *attr)
{
+ struct hns_roce_create_cq_ex_resp resp_ex = {};
struct hns_roce_ib_create_cq_resp *resp_drv;
- struct hns_roce_create_cq_resp resp = {};
+ struct hns_roce_create_cq_ex cmd_ex = {};
struct hns_roce_ib_create_cq *cmd_drv;
- struct hns_roce_create_cq cmd = {};
int ret;
- cmd_drv = &cmd.drv_payload;
- resp_drv = &resp.drv_payload;
+ cmd_drv = &cmd_ex.drv_payload;
+ resp_drv = &resp_ex.drv_payload;
cmd_drv->buf_addr = (uintptr_t)cq->buf.buf;
cmd_drv->db_addr = (uintptr_t)cq->db;
cmd_drv->cqe_size = (uintptr_t)cq->cqe_size;
- ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
- &cq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd),
- &resp.ibv_resp, sizeof(resp));
+ ret = ibv_cmd_create_cq_ex(context, attr, &cq->verbs_cq,
+ &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ &resp_ex.ibv_resp, sizeof(resp_ex), 0);
if (ret)
return ret;
@@ -325,16 +330,15 @@ static int exec_cq_create_cmd(struct ibv_context *context,
return 0;
}
-struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
- struct ibv_comp_channel *channel,
- int comp_vector)
+static struct ibv_cq_ex *create_cq(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *attr)
{
struct hns_roce_device *hr_dev = to_hr_dev(context->device);
struct hns_roce_context *hr_ctx = to_hr_ctx(context);
struct hns_roce_cq *cq;
int ret;
- ret = hns_roce_verify_cq(&cqe, hr_ctx);
+ ret = verify_cq_create_attr(attr, hr_ctx);
if (ret)
goto err;
@@ -348,7 +352,7 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
if (ret)
goto err_lock;
- cq->cq_depth = cqe;
+ cq->cq_depth = attr->cqe;
cq->cqe_size = hr_ctx->cqe_size;
ret = hns_roce_alloc_cq_buf(cq);
@@ -363,13 +367,13 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
*cq->db = 0;
- ret = exec_cq_create_cmd(context, cq, cqe, channel, comp_vector);
+ ret = exec_cq_create_cmd(context, cq, attr);
if (ret)
goto err_cmd;
cq->arm_sn = 1;
- return &cq->ibv_cq;
+ return &cq->verbs_cq.cq_ex;
err_cmd:
if (hr_dev->hw_version != HNS_ROCE_HW_VER1)
@@ -387,6 +391,27 @@ err:
return NULL;
}
+struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector)
+{
+ struct ibv_cq_ex *cq;
+ struct ibv_cq_init_attr_ex attr = {
+ .cqe = cqe,
+ .channel = channel,
+ .comp_vector = comp_vector,
+ };
+
+ cq = create_cq(context, &attr);
+ return cq ? ibv_cq_ex_to_cq(cq) : NULL;
+}
+
+struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *attr)
+{
+ return create_cq(context, attr);
+}
+
void hns_roce_u_cq_event(struct ibv_cq *cq)
{
to_hr_cq(cq)->arm_sn++;
--
2.30.0

415
0041-libhns-Extended-CQ-supports-the-new-polling-mechanis.patch Normal file
View File

@ -0,0 +1,415 @@
From 0464e0cb0416d679aba3b58261bbd2cadb74fd03 Mon Sep 17 00:00:00 2001
From: Xinhao Liu <liuxinhao5@hisilicon.com>
Date: Mon, 7 Mar 2022 18:49:36 +0800
Subject: libhns: Extended CQ supports the new polling mechanism
ofed provides new polling APIs for extended CQ. With the new APIs, users
can poll the extended CQ more efficiently.
Signed-off-by: Xinhao Liu <liuxinhao5@hisilicon.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u.h | 1 +
providers/hns/hns_roce_u_hw_v2.c | 319 +++++++++++++++++++++++++++++++
providers/hns/hns_roce_u_hw_v2.h | 1 +
providers/hns/hns_roce_u_verbs.c | 18 +-
4 files changed, 337 insertions(+), 2 deletions(-)
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 505e7498..70ac6e5b 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -247,6 +247,7 @@ struct hns_roce_cq {
int arm_sn;
unsigned long flags;
unsigned int cqe_size;
+ struct hns_roce_v2_cqe *cqe;
};
struct hns_roce_idx_que {
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 07f3596d..081ab1f3 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -1657,6 +1657,325 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
return ret;
}
+static void cqe_proc_sq(struct hns_roce_qp *hr_qp, uint32_t wqe_idx,
+ struct hns_roce_cq *cq)
+{
+ struct hns_roce_wq *wq = &hr_qp->sq;
+
+ if (hr_qp->sq_signal_bits)
+ wq->tail += (wqe_idx - wq->tail) & (wq->wqe_cnt - 1);
+
+ cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+ ++wq->tail;
+}
+
+static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx,
+ struct hns_roce_cq *cq)
+{
+ cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)];
+ hns_roce_free_srq_wqe(srq, wqe_idx);
+}
+
+static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq)
+{
+ struct hns_roce_wq *wq = &hr_qp->rq;
+
+ cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+ ++wq->tail;
+}
+
+static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp,
+ struct hns_roce_cq *cq)
+{
+ struct hns_roce_v2_cqe *cqe = cq->cqe;
+ struct hns_roce_srq *srq = NULL;
+ uint32_t wqe_idx;
+
+ wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
+ if (hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ) {
+ cqe_proc_sq(qp, wqe_idx, cq);
+ } else {
+ if (get_srq_from_cqe(cqe, ctx, qp, &srq))
+ return V2_CQ_POLL_ERR;
+
+ if (srq)
+ cqe_proc_srq(srq, wqe_idx, cq);
+ else
+ cqe_proc_rq(qp, cq);
+ }
+ return 0;
+}
+
+static void handle_error_cqe_ex(struct hns_roce_cq *cq, uint8_t status)
+{
+ int i;
+
+ static const struct {
+ unsigned int cqe_status;
+ enum ibv_wc_status wc_status;
+ } map[] = {
+ { HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR, IBV_WC_LOC_LEN_ERR },
+ { HNS_ROCE_V2_CQE_LOCAL_QP_OP_ERR, IBV_WC_LOC_QP_OP_ERR },
+ { HNS_ROCE_V2_CQE_LOCAL_PROT_ERR, IBV_WC_LOC_PROT_ERR },
+ { HNS_ROCE_V2_CQE_WR_FLUSH_ERR, IBV_WC_WR_FLUSH_ERR },
+ { HNS_ROCE_V2_CQE_MEM_MANAGERENT_OP_ERR, IBV_WC_MW_BIND_ERR },
+ { HNS_ROCE_V2_CQE_BAD_RESP_ERR, IBV_WC_BAD_RESP_ERR },
+ { HNS_ROCE_V2_CQE_LOCAL_ACCESS_ERR, IBV_WC_LOC_ACCESS_ERR },
+ { HNS_ROCE_V2_CQE_REMOTE_INVAL_REQ_ERR, IBV_WC_REM_INV_REQ_ERR },
+ { HNS_ROCE_V2_CQE_REMOTE_ACCESS_ERR, IBV_WC_REM_ACCESS_ERR },
+ { HNS_ROCE_V2_CQE_REMOTE_OP_ERR, IBV_WC_REM_OP_ERR },
+ { HNS_ROCE_V2_CQE_TRANSPORT_RETRY_EXC_ERR,
+ IBV_WC_RETRY_EXC_ERR },
+ { HNS_ROCE_V2_CQE_RNR_RETRY_EXC_ERR, IBV_WC_RNR_RETRY_EXC_ERR },
+ { HNS_ROCE_V2_CQE_REMOTE_ABORTED_ERR, IBV_WC_REM_ABORT_ERR },
+ { HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR },
+ };
+
+ cq->verbs_cq.cq_ex.status = IBV_WC_GENERAL_ERR;
+ for (i = 0; i < ARRAY_SIZE(map); i++) {
+ if (status == map[i].cqe_status) {
+ cq->verbs_cq.cq_ex.status = map[i].wc_status;
+ break;
+ }
+ }
+}
+
+static int wc_poll_cqe(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
+{
+ struct hns_roce_qp *qp = NULL;
+ struct hns_roce_v2_cqe *cqe;
+ uint8_t status;
+ uint32_t qpn;
+
+ cqe = next_cqe_sw_v2(cq);
+ if (!cqe)
+ return ENOENT;
+
+ ++cq->cons_index;
+ udma_from_device_barrier();
+
+ cq->cqe = cqe;
+ qpn = hr_reg_read(cqe, CQE_LCL_QPN);
+
+ qp = hns_roce_v2_find_qp(ctx, qpn);
+ if (!qp)
+ return V2_CQ_POLL_ERR;
+
+ if (cqe_proc_wq(ctx, qp, cq))
+ return V2_CQ_POLL_ERR;
+
+ status = hr_reg_read(cqe, CQE_STATUS);
+
+ /*
+ * once a cqe in error status, the driver needs to help the HW to
+ * generated flushed cqes for all subsequent wqes
+ */
+ if (status != HNS_ROCE_V2_CQE_SUCCESS) {
+ handle_error_cqe_ex(cq, status);
+ return hns_roce_flush_cqe(qp, status);
+ }
+
+ cq->verbs_cq.cq_ex.status = IBV_WC_SUCCESS;
+
+ return V2_CQ_OK;
+}
+
+static int wc_start_poll_cq(struct ibv_cq_ex *current,
+ struct ibv_poll_cq_attr *attr)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ struct hns_roce_context *ctx = to_hr_ctx(current->context);
+ int err;
+
+ if (attr->comp_mask)
+ return EINVAL;
+
+ pthread_spin_lock(&cq->lock);
+
+ err = wc_poll_cqe(ctx, cq);
+ if (err != V2_CQ_OK)
+ pthread_spin_unlock(&cq->lock);
+
+ return err;
+}
+
+static int wc_next_poll_cq(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ struct hns_roce_context *ctx = to_hr_ctx(current->context);
+ int err;
+
+ err = wc_poll_cqe(ctx, cq);
+ if (err != V2_CQ_OK)
+ return err;
+
+ if (cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)
+ *cq->db = cq->cons_index & RECORD_DB_CI_MASK;
+ else
+ update_cq_db(ctx, cq);
+
+ return 0;
+}
+
+static void wc_end_poll_cq(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ struct hns_roce_context *ctx = to_hr_ctx(current->context);
+
+ if (cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)
+ *cq->db = cq->cons_index & RECORD_DB_CI_MASK;
+ else
+ update_cq_db(ctx, cq);
+
+ pthread_spin_unlock(&cq->lock);
+}
+
+static enum ibv_wc_opcode wc_read_opcode(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ uint8_t opcode = hr_reg_read(cq->cqe, CQE_OPCODE);
+
+ if (hr_reg_read(cq->cqe, CQE_S_R) == CQE_FOR_SQ)
+ return wc_send_op_map[opcode];
+ else
+ return wc_rcv_op_map[opcode];
+}
+
+static uint32_t wc_read_vendor_err(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return hr_reg_read(cq->cqe, CQE_SUB_STATUS);
+}
+
+static uint32_t wc_read_byte_len(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return le32toh(cq->cqe->byte_cnt);
+}
+
+static __be32 wc_read_imm_data(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ if (hr_reg_read(cq->cqe, CQE_OPCODE) == HNS_ROCE_RECV_OP_SEND_WITH_INV)
+ /* This is returning invalidate_rkey which is in host order, see
+ * ibv_wc_read_invalidated_rkey.
+ */
+ return (__force __be32)le32toh(cq->cqe->rkey);
+
+ return htobe32(le32toh(cq->cqe->immtdata));
+}
+
+static uint32_t wc_read_qp_num(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return hr_reg_read(cq->cqe, CQE_LCL_QPN);
+}
+
+static uint32_t wc_read_src_qp(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return hr_reg_read(cq->cqe, CQE_RMT_QPN);
+}
+
+static unsigned int get_wc_flags_for_sq(uint8_t opcode)
+{
+ switch (opcode) {
+ case HNS_ROCE_SQ_OP_SEND_WITH_IMM:
+ case HNS_ROCE_SQ_OP_RDMA_WRITE_WITH_IMM:
+ return IBV_WC_WITH_IMM;
+ case HNS_ROCE_SQ_OP_LOCAL_INV:
+ return IBV_WC_WITH_INV;
+ default:
+ return 0;
+ }
+}
+
+static unsigned int get_wc_flags_for_rq(uint8_t opcode)
+{
+ switch (opcode) {
+ case HNS_ROCE_RECV_OP_RDMA_WRITE_IMM:
+ case HNS_ROCE_RECV_OP_SEND_WITH_IMM:
+ return IBV_WC_WITH_IMM;
+ case HNS_ROCE_RECV_OP_SEND_WITH_INV:
+ return IBV_WC_WITH_INV;
+ default:
+ return 0;
+ }
+}
+
+static unsigned int wc_read_wc_flags(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+ uint8_t opcode = hr_reg_read(cq->cqe, CQE_OPCODE);
+ unsigned int wc_flags;
+
+ if (hr_reg_read(cq->cqe, CQE_S_R) == CQE_FOR_SQ) {
+ wc_flags = get_wc_flags_for_sq(opcode);
+ } else {
+ wc_flags = get_wc_flags_for_rq(opcode);
+ wc_flags |= hr_reg_read(cq->cqe, CQE_GRH) ? IBV_WC_GRH : 0;
+ }
+
+ return wc_flags;
+}
+
+static uint32_t wc_read_slid(struct ibv_cq_ex *current)
+{
+ return 0;
+}
+
+static uint8_t wc_read_sl(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return (uint8_t)hr_reg_read(cq->cqe, CQE_SL);
+}
+
+static uint8_t wc_read_dlid_path_bits(struct ibv_cq_ex *current)
+{
+ return 0;
+}
+
+static uint16_t wc_read_cvlan(struct ibv_cq_ex *current)
+{
+ struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
+
+ return hr_reg_read(cq->cqe, CQE_VID_VLD) ?
+ hr_reg_read(cq->cqe, CQE_VID) : 0;
+}
+
+void hns_roce_attach_cq_ex_ops(struct ibv_cq_ex *cq_ex, uint64_t wc_flags)
+{
+ cq_ex->start_poll = wc_start_poll_cq;
+ cq_ex->next_poll = wc_next_poll_cq;
+ cq_ex->end_poll = wc_end_poll_cq;
+ cq_ex->read_opcode = wc_read_opcode;
+ cq_ex->read_vendor_err = wc_read_vendor_err;
+ cq_ex->read_wc_flags = wc_read_wc_flags;
+
+ if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
+ cq_ex->read_byte_len = wc_read_byte_len;
+ if (wc_flags & IBV_WC_EX_WITH_IMM)
+ cq_ex->read_imm_data = wc_read_imm_data;
+ if (wc_flags & IBV_WC_EX_WITH_QP_NUM)
+ cq_ex->read_qp_num = wc_read_qp_num;
+ if (wc_flags & IBV_WC_EX_WITH_SRC_QP)
+ cq_ex->read_src_qp = wc_read_src_qp;
+ if (wc_flags & IBV_WC_EX_WITH_SLID)
+ cq_ex->read_slid = wc_read_slid;
+ if (wc_flags & IBV_WC_EX_WITH_SL)
+ cq_ex->read_sl = wc_read_sl;
+ if (wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
+ cq_ex->read_dlid_path_bits = wc_read_dlid_path_bits;
+ if (wc_flags & IBV_WC_EX_WITH_CVLAN)
+ cq_ex->read_cvlan = wc_read_cvlan;
+}
+
const struct hns_roce_u_hw hns_roce_u_hw_v2 = {
.hw_version = HNS_ROCE_HW_VER2,
.hw_ops = {
diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h
index 92e5f1a4..0068f4fe 100644
--- a/providers/hns/hns_roce_u_hw_v2.h
+++ b/providers/hns/hns_roce_u_hw_v2.h
@@ -337,5 +337,6 @@ struct hns_roce_ud_sq_wqe {
#define MAX_SERVICE_LEVEL 0x7
void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp);
+void hns_roce_attach_cq_ex_ops(struct ibv_cq_ex *cq_ex, uint64_t wc_flags);
#endif /* _HNS_ROCE_U_HW_V2_H */
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index a993c39a..9ea8a6d3 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -276,13 +276,21 @@ int hns_roce_u_dealloc_mw(struct ibv_mw *mw)
return 0;
}
+enum {
+ CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS |
+ IBV_WC_EX_WITH_CVLAN,
+};
+
static int verify_cq_create_attr(struct ibv_cq_init_attr_ex *attr,
struct hns_roce_context *context)
{
if (!attr->cqe || attr->cqe > context->max_cqe)
return -EINVAL;
- if (attr->comp_mask || attr->wc_flags)
+ if (attr->comp_mask)
+ return -EOPNOTSUPP;
+
+ if (!check_comp_mask(attr->wc_flags, CREATE_CQ_SUPPORTED_WC_FLAGS))
return -EOPNOTSUPP;
attr->cqe = max_t(uint32_t, HNS_ROCE_MIN_CQE_NUM,
@@ -409,7 +417,13 @@ struct ibv_cq *hns_roce_u_create_cq(struct ibv_context *context, int cqe,
struct ibv_cq_ex *hns_roce_u_create_cq_ex(struct ibv_context *context,
struct ibv_cq_init_attr_ex *attr)
{
- return create_cq(context, attr);
+ struct ibv_cq_ex *cq;
+
+ cq = create_cq(context, attr);
+ if (cq)
+ hns_roce_attach_cq_ex_ops(cq, attr->wc_flags);
+
+ return cq;
}
void hns_roce_u_cq_event(struct ibv_cq *cq)
--
2.30.0

160
0042-libhns-Optimize-the-error-handling-of-CQE.patch Normal file
View File

@ -0,0 +1,160 @@
From 2d48954e9b2617cb48f7d5ba47a10ceda4e556ff Mon Sep 17 00:00:00 2001
From: Xinhao Liu <liuxinhao5@hisilicon.com>
Date: Mon, 7 Mar 2022 18:49:37 +0800
Subject: libhns: Optimize the error handling of CQE
Separate the acquisition of wc->status and wc->vendor_err to make the logic
of error handling clearer.
Signed-off-by: Xinhao Liu <liuxinhao5@hisilicon.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u_hw_v2.c | 81 ++++++++------------------------
1 file changed, 19 insertions(+), 62 deletions(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 081ab1f3..2804450d 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -146,13 +146,13 @@ static int set_atomic_seg(struct hns_roce_qp *qp, struct ibv_send_wr *wr,
return 0;
}
-static void handle_error_cqe(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
- uint8_t status)
+static enum ibv_wc_status get_wc_status(uint8_t status)
{
static const struct {
unsigned int cqe_status;
enum ibv_wc_status wc_status;
} map[] = {
+ { HNS_ROCE_V2_CQE_SUCCESS, IBV_WC_SUCCESS },
{ HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR, IBV_WC_LOC_LEN_ERR },
{ HNS_ROCE_V2_CQE_LOCAL_QP_OP_ERR, IBV_WC_LOC_QP_OP_ERR },
{ HNS_ROCE_V2_CQE_LOCAL_PROT_ERR, IBV_WC_LOC_PROT_ERR },
@@ -169,17 +169,12 @@ static void handle_error_cqe(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
{ HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR },
};
- int i;
-
- wc->status = IBV_WC_GENERAL_ERR;
- for (i = 0; i < ARRAY_SIZE(map); i++) {
- if (status == map[i].cqe_status) {
- wc->status = map[i].wc_status;
- break;
- }
+ for (int i = 0; i < ARRAY_SIZE(map); i++) {
+ if (status == map[i].cqe_status)
+ return map[i].wc_status;
}
- wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
+ return IBV_WC_GENERAL_ERR;
}
static struct hns_roce_v2_cqe *get_cqe_v2(struct hns_roce_cq *cq, int entry)
@@ -581,7 +576,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
return V2_CQ_POLL_ERR;
}
- status = hr_reg_read(cqe, CQE_STATUS);
opcode = hr_reg_read(cqe, CQE_OPCODE);
is_send = hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ;
if (is_send) {
@@ -603,18 +597,18 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
wc->qp_num = qpn;
+ status = hr_reg_read(cqe, CQE_STATUS);
+ wc->status = get_wc_status(status);
+ wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
+
+ if (status == HNS_ROCE_V2_CQE_SUCCESS)
+ return V2_CQ_OK;
+
/*
* once a cqe in error status, the driver needs to help the HW to
* generated flushed cqes for all subsequent wqes
*/
- if (status != HNS_ROCE_V2_CQE_SUCCESS) {
- handle_error_cqe(cqe, wc, status);
- return hns_roce_flush_cqe(*cur_qp, status);
- }
-
- wc->status = IBV_WC_SUCCESS;
-
- return V2_CQ_OK;
+ return hns_roce_flush_cqe(*cur_qp, status);
}
static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,
@@ -1706,40 +1700,6 @@ static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp,
return 0;
}
-static void handle_error_cqe_ex(struct hns_roce_cq *cq, uint8_t status)
-{
- int i;
-
- static const struct {
- unsigned int cqe_status;
- enum ibv_wc_status wc_status;
- } map[] = {
- { HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR, IBV_WC_LOC_LEN_ERR },
- { HNS_ROCE_V2_CQE_LOCAL_QP_OP_ERR, IBV_WC_LOC_QP_OP_ERR },
- { HNS_ROCE_V2_CQE_LOCAL_PROT_ERR, IBV_WC_LOC_PROT_ERR },
- { HNS_ROCE_V2_CQE_WR_FLUSH_ERR, IBV_WC_WR_FLUSH_ERR },
- { HNS_ROCE_V2_CQE_MEM_MANAGERENT_OP_ERR, IBV_WC_MW_BIND_ERR },
- { HNS_ROCE_V2_CQE_BAD_RESP_ERR, IBV_WC_BAD_RESP_ERR },
- { HNS_ROCE_V2_CQE_LOCAL_ACCESS_ERR, IBV_WC_LOC_ACCESS_ERR },
- { HNS_ROCE_V2_CQE_REMOTE_INVAL_REQ_ERR, IBV_WC_REM_INV_REQ_ERR },
- { HNS_ROCE_V2_CQE_REMOTE_ACCESS_ERR, IBV_WC_REM_ACCESS_ERR },
- { HNS_ROCE_V2_CQE_REMOTE_OP_ERR, IBV_WC_REM_OP_ERR },
- { HNS_ROCE_V2_CQE_TRANSPORT_RETRY_EXC_ERR,
- IBV_WC_RETRY_EXC_ERR },
- { HNS_ROCE_V2_CQE_RNR_RETRY_EXC_ERR, IBV_WC_RNR_RETRY_EXC_ERR },
- { HNS_ROCE_V2_CQE_REMOTE_ABORTED_ERR, IBV_WC_REM_ABORT_ERR },
- { HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR },
- };
-
- cq->verbs_cq.cq_ex.status = IBV_WC_GENERAL_ERR;
- for (i = 0; i < ARRAY_SIZE(map); i++) {
- if (status == map[i].cqe_status) {
- cq->verbs_cq.cq_ex.status = map[i].wc_status;
- break;
- }
- }
-}
-
static int wc_poll_cqe(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
{
struct hns_roce_qp *qp = NULL;
@@ -1765,19 +1725,16 @@ static int wc_poll_cqe(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
return V2_CQ_POLL_ERR;
status = hr_reg_read(cqe, CQE_STATUS);
+ cq->verbs_cq.cq_ex.status = get_wc_status(status);
+
+ if (status == HNS_ROCE_V2_CQE_SUCCESS)
+ return V2_CQ_OK;
/*
* once a cqe in error status, the driver needs to help the HW to
* generated flushed cqes for all subsequent wqes
*/
- if (status != HNS_ROCE_V2_CQE_SUCCESS) {
- handle_error_cqe_ex(cq, status);
- return hns_roce_flush_cqe(qp, status);
- }
-
- cq->verbs_cq.cq_ex.status = IBV_WC_SUCCESS;
-
- return V2_CQ_OK;
+ return hns_roce_flush_cqe(qp, status);
}
static int wc_start_poll_cq(struct ibv_cq_ex *current,
--
2.30.0

323
0043-libhns-Refactor-hns-roce-v2-poll-one-and-wc-poll-cqe.patch Normal file
View File

@ -0,0 +1,323 @@
From 9dd7b55957ccc720a6844613af9d43680d8fbaad Mon Sep 17 00:00:00 2001
From: Xinhao Liu <liuxinhao5@hisilicon.com>
Date: Mon, 7 Mar 2022 18:49:38 +0800
Subject: libhns: Refactor hns roce v2 poll one() and wc poll cqe()
hns_roce_v2_poll_one() and wc_poll_cqe() have a lot of repetitive code.
Aggregating the repetitive parts of these two functions into one function
hns_roce_poll_one() can reduce the repetitive code.
Signed-off-by: Xinhao Liu <liuxinhao5@hisilicon.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u_hw_v2.c | 228 +++++++++++++++----------------
1 file changed, 107 insertions(+), 121 deletions(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 2804450d..42a77151 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -285,6 +285,7 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx,
static void hns_roce_update_sq_db(struct hns_roce_context *ctx,
struct hns_roce_qp *qp)
+
{
struct hns_roce_db sq_db = {};
@@ -548,21 +549,101 @@ static void parse_cqe_for_req(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
wc->opcode = wc_send_op_map[opcode];
}
-static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
- struct hns_roce_qp **cur_qp, struct ibv_wc *wc)
+static void cqe_proc_sq(struct hns_roce_qp *hr_qp, uint32_t wqe_idx,
+ struct hns_roce_cq *cq)
{
- struct hns_roce_context *ctx = to_hr_ctx(cq->verbs_cq.cq.context);
+ struct hns_roce_wq *wq = &hr_qp->sq;
+
+ if (hr_qp->sq_signal_bits)
+ wq->tail += (wqe_idx - wq->tail) & (wq->wqe_cnt - 1);
+
+ cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+ ++wq->tail;
+}
+
+static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx,
+ struct hns_roce_cq *cq)
+{
+ cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)];
+ hns_roce_free_srq_wqe(srq, wqe_idx);
+}
+
+static void cqe_proc_rq(struct hns_roce_wq *wq, struct hns_roce_cq *cq)
+{
+ cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+ ++wq->tail;
+}
+
+static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp,
+ struct hns_roce_cq *cq)
+{
+ struct hns_roce_v2_cqe *cqe = cq->cqe;
+ struct hns_roce_srq *srq = NULL;
+ uint32_t wqe_idx;
+
+ wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
+ if (hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ) {
+ cqe_proc_sq(qp, wqe_idx, cq);
+ } else {
+ if (get_srq_from_cqe(cqe, ctx, qp, &srq))
+ return V2_CQ_POLL_ERR;
+
+ if (srq)
+ cqe_proc_srq(srq, wqe_idx, cq);
+ else
+ cqe_proc_rq(&qp->rq, cq);
+ }
+
+ return 0;
+}
+
+static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq,
+ struct hns_roce_qp *cur_qp, struct ibv_wc *wc)
+{
+ struct hns_roce_v2_cqe *cqe = cq->cqe;
struct hns_roce_srq *srq = NULL;
- struct hns_roce_v2_cqe *cqe;
uint8_t opcode;
- uint8_t status;
+
+ if (!wc) {
+ if (cqe_proc_wq(ctx, cur_qp, cq))
+ return V2_CQ_POLL_ERR;
+
+ return 0;
+ }
+
+ opcode = hr_reg_read(cqe, CQE_OPCODE);
+
+ if (hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ) {
+ parse_cqe_for_req(cqe, wc, cur_qp, opcode);
+ } else {
+ wc->byte_len = le32toh(cqe->byte_cnt);
+ get_opcode_for_resp(cqe, wc, opcode);
+
+ if (get_srq_from_cqe(cqe, ctx, cur_qp, &srq))
+ return V2_CQ_POLL_ERR;
+
+ if (srq)
+ parse_cqe_for_srq(cqe, wc, srq);
+ else
+ parse_cqe_for_resp(cqe, wc, cur_qp, opcode);
+ }
+
+ return 0;
+}
+
+static int hns_roce_poll_one(struct hns_roce_context *ctx,
+ struct hns_roce_qp **cur_qp, struct hns_roce_cq *cq,
+ struct ibv_wc *wc)
+{
+ struct hns_roce_v2_cqe *cqe;
+ uint8_t status, wc_status;
uint32_t qpn;
- bool is_send;
cqe = next_cqe_sw_v2(cq);
if (!cqe)
- return V2_CQ_EMPTY;
+ return wc ? V2_CQ_EMPTY : ENOENT;
+ cq->cqe = cqe;
++cq->cons_index;
udma_from_device_barrier();
@@ -576,31 +657,20 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
return V2_CQ_POLL_ERR;
}
- opcode = hr_reg_read(cqe, CQE_OPCODE);
- is_send = hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ;
- if (is_send) {
- parse_cqe_for_req(cqe, wc, *cur_qp, opcode);
- } else {
- wc->byte_len = le32toh(cqe->byte_cnt);
- get_opcode_for_resp(cqe, wc, opcode);
+ if (parse_cqe_for_cq(ctx, cq, *cur_qp, wc))
+ return V2_CQ_POLL_ERR;
- if (get_srq_from_cqe(cqe, ctx, *cur_qp, &srq))
- return V2_CQ_POLL_ERR;
+ status = hr_reg_read(cqe, CQE_STATUS);
+ wc_status = get_wc_status(status);
- if (srq) {
- parse_cqe_for_srq(cqe, wc, srq);
- } else {
- if (parse_cqe_for_resp(cqe, wc, *cur_qp, opcode))
- return V2_CQ_POLL_ERR;
- }
+ if (wc) {
+ wc->status = wc_status;
+ wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
+ wc->qp_num = qpn;
+ } else {
+ cq->verbs_cq.cq_ex.status = wc_status;
}
- wc->qp_num = qpn;
-
- status = hr_reg_read(cqe, CQE_STATUS);
- wc->status = get_wc_status(status);
- wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
-
if (status == HNS_ROCE_V2_CQE_SUCCESS)
return V2_CQ_OK;
@@ -614,16 +684,16 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *cq,
static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne,
struct ibv_wc *wc)
{
- int npolled;
- int err = V2_CQ_OK;
- struct hns_roce_qp *qp = NULL;
- struct hns_roce_cq *cq = to_hr_cq(ibvcq);
struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context);
+ struct hns_roce_cq *cq = to_hr_cq(ibvcq);
+ struct hns_roce_qp *qp = NULL;
+ int err = V2_CQ_OK;
+ int npolled;
pthread_spin_lock(&cq->lock);
for (npolled = 0; npolled < ne; ++npolled) {
- err = hns_roce_v2_poll_one(cq, &qp, wc + npolled);
+ err = hns_roce_poll_one(ctx, &qp, cq, wc + npolled);
if (err != V2_CQ_OK)
break;
}
@@ -1651,97 +1721,12 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq,
return ret;
}
-static void cqe_proc_sq(struct hns_roce_qp *hr_qp, uint32_t wqe_idx,
- struct hns_roce_cq *cq)
-{
- struct hns_roce_wq *wq = &hr_qp->sq;
-
- if (hr_qp->sq_signal_bits)
- wq->tail += (wqe_idx - wq->tail) & (wq->wqe_cnt - 1);
-
- cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
- ++wq->tail;
-}
-
-static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx,
- struct hns_roce_cq *cq)
-{
- cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)];
- hns_roce_free_srq_wqe(srq, wqe_idx);
-}
-
-static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq)
-{
- struct hns_roce_wq *wq = &hr_qp->rq;
-
- cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
- ++wq->tail;
-}
-
-static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp,
- struct hns_roce_cq *cq)
-{
- struct hns_roce_v2_cqe *cqe = cq->cqe;
- struct hns_roce_srq *srq = NULL;
- uint32_t wqe_idx;
-
- wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
- if (hr_reg_read(cqe, CQE_S_R) == CQE_FOR_SQ) {
- cqe_proc_sq(qp, wqe_idx, cq);
- } else {
- if (get_srq_from_cqe(cqe, ctx, qp, &srq))
- return V2_CQ_POLL_ERR;
-
- if (srq)
- cqe_proc_srq(srq, wqe_idx, cq);
- else
- cqe_proc_rq(qp, cq);
- }
- return 0;
-}
-
-static int wc_poll_cqe(struct hns_roce_context *ctx, struct hns_roce_cq *cq)
-{
- struct hns_roce_qp *qp = NULL;
- struct hns_roce_v2_cqe *cqe;
- uint8_t status;
- uint32_t qpn;
-
- cqe = next_cqe_sw_v2(cq);
- if (!cqe)
- return ENOENT;
-
- ++cq->cons_index;
- udma_from_device_barrier();
-
- cq->cqe = cqe;
- qpn = hr_reg_read(cqe, CQE_LCL_QPN);
-
- qp = hns_roce_v2_find_qp(ctx, qpn);
- if (!qp)
- return V2_CQ_POLL_ERR;
-
- if (cqe_proc_wq(ctx, qp, cq))
- return V2_CQ_POLL_ERR;
-
- status = hr_reg_read(cqe, CQE_STATUS);
- cq->verbs_cq.cq_ex.status = get_wc_status(status);
-
- if (status == HNS_ROCE_V2_CQE_SUCCESS)
- return V2_CQ_OK;
-
- /*
- * once a cqe in error status, the driver needs to help the HW to
- * generated flushed cqes for all subsequent wqes
- */
- return hns_roce_flush_cqe(qp, status);
-}
-
static int wc_start_poll_cq(struct ibv_cq_ex *current,
struct ibv_poll_cq_attr *attr)
{
struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
struct hns_roce_context *ctx = to_hr_ctx(current->context);
+ struct hns_roce_qp *qp = NULL;
int err;
if (attr->comp_mask)
@@ -1749,7 +1734,7 @@ static int wc_start_poll_cq(struct ibv_cq_ex *current,
pthread_spin_lock(&cq->lock);
- err = wc_poll_cqe(ctx, cq);
+ err = hns_roce_poll_one(ctx, &qp, cq, NULL);
if (err != V2_CQ_OK)
pthread_spin_unlock(&cq->lock);
@@ -1760,9 +1745,10 @@ static int wc_next_poll_cq(struct ibv_cq_ex *current)
{
struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current));
struct hns_roce_context *ctx = to_hr_ctx(current->context);
+ struct hns_roce_qp *qp = NULL;
int err;
- err = wc_poll_cqe(ctx, cq);
+ err = hns_roce_poll_one(ctx, &qp, cq, NULL);
if (err != V2_CQ_OK)
return err;
--
2.30.0

1007
0044-libhns-Extended-QP-supports-the-new-post-send-mechan.patch Normal file
View File

File diff suppressed because it is too large Load Diff

58
0045-libhns-Add-general-error-type-for-CQE.patch Normal file
View File

@ -0,0 +1,58 @@
From 1e5f8bb89169453cfdd17bf58cef7186dcf58596 Mon Sep 17 00:00:00 2001
From: Youming Luo <luoyouming@huawei.com>
Date: Wed, 16 Mar 2022 17:36:39 +0800
Subject: libhns: Add general error type for CQE
If a Work Request posted in an RQ of UD QP isn't big enough for holding the
incoming message, then the hns ROCEE will generate a general error CQE. The
IB specification does not specify this type of CQE.
In the case of unreliable communication, it is not desirable to change the
QP to an error state due to an insufficient receive length error. So If the
hns ROCEE reports a general error CQE, it's no need to set the QP to an
error state, and the driver should skip it.
Signed-off-by: Youming Luo <luoyouming@huawei.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u_hw_v2.c | 4 +++-
providers/hns/hns_roce_u_hw_v2.h | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 42a77151..fab1939b 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -166,6 +166,7 @@ static enum ibv_wc_status get_wc_status(uint8_t status)
{ HNS_ROCE_V2_CQE_TRANSPORT_RETRY_EXC_ERR, IBV_WC_RETRY_EXC_ERR },
{ HNS_ROCE_V2_CQE_RNR_RETRY_EXC_ERR, IBV_WC_RNR_RETRY_EXC_ERR },
{ HNS_ROCE_V2_CQE_REMOTE_ABORTED_ERR, IBV_WC_REM_ABORT_ERR },
+ { HNS_ROCE_V2_CQE_GENERAL_ERR, IBV_WC_GENERAL_ERR },
{ HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR, IBV_WC_REM_INV_RD_REQ_ERR },
};
@@ -671,7 +672,8 @@ static int hns_roce_poll_one(struct hns_roce_context *ctx,
cq->verbs_cq.cq_ex.status = wc_status;
}
- if (status == HNS_ROCE_V2_CQE_SUCCESS)
+ if (status == HNS_ROCE_V2_CQE_SUCCESS ||
+ status == HNS_ROCE_V2_CQE_GENERAL_ERR)
return V2_CQ_OK;
/*
diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h
index 0068f4fe..122fdbdf 100644
--- a/providers/hns/hns_roce_u_hw_v2.h
+++ b/providers/hns/hns_roce_u_hw_v2.h
@@ -110,6 +110,7 @@ enum {
HNS_ROCE_V2_CQE_TRANSPORT_RETRY_EXC_ERR = 0x15,
HNS_ROCE_V2_CQE_RNR_RETRY_EXC_ERR = 0x16,
HNS_ROCE_V2_CQE_REMOTE_ABORTED_ERR = 0x22,
+ HNS_ROCE_V2_CQE_GENERAL_ERR = 0x23,
HNS_ROCE_V2_CQE_XRC_VIOLATION_ERR = 0x24,
};
--
2.30.0

14
rdma-core.spec
View File

@ -1,6 +1,6 @@
Name: rdma-core Name: rdma-core
Version: 35.1 Version: 35.1
Release: 5 Release: 6
Summary: RDMA core userspace libraries and daemons Summary: RDMA core userspace libraries and daemons
License: GPLv2 or BSD License: GPLv2 or BSD
Url: https://github.com/linux-rdma/rdma-core Url: https://github.com/linux-rdma/rdma-core
@ -46,6 +46,12 @@ Patch36: 0036-libhns-Fix-the-calculation-of-QP-SRQ-table-size.patch
Patch37: 0037-libhns-Fix-wrong-HIP08-version-macro.patch Patch37: 0037-libhns-Fix-wrong-HIP08-version-macro.patch
Patch38: 0038-libhns-Fix-out-of-bounds-write-when-filling-inline-d.patch Patch38: 0038-libhns-Fix-out-of-bounds-write-when-filling-inline-d.patch
Patch39: 0039-libhns-Clear-remaining-unused-sges-when-post-recv.patch Patch39: 0039-libhns-Clear-remaining-unused-sges-when-post-recv.patch
Patch40: 0040-libhns-Add-support-for-creating-extended-CQ.patch
Patch41: 0041-libhns-Extended-CQ-supports-the-new-polling-mechanis.patch
Patch42: 0042-libhns-Optimize-the-error-handling-of-CQE.patch
Patch43: 0043-libhns-Refactor-hns-roce-v2-poll-one-and-wc-poll-cqe.patch
Patch44: 0044-libhns-Extended-QP-supports-the-new-post-send-mechan.patch
Patch45: 0045-libhns-Add-general-error-type-for-CQE.patch
BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0)
BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel
@ -290,6 +296,12 @@ fi
%{_mandir}/* %{_mandir}/*
%changelog %changelog
* Wed Aug 24 2022 luozhengfeng <luozhengfeng@h-partners.com> - 35.1-6
- Type: requirement
- ID: NA
- SUG: NA
- DESC: Add support for hns CQ_EX
* Fri Aug 12 2022 luozhengfeng <luozhengfeng@h-partners.com> - 35.1-5 * Fri Aug 12 2022 luozhengfeng <luozhengfeng@h-partners.com> - 35.1-5
- Type: bugfix - Type: bugfix
- ID: NA - ID: NA