From 0025a5cf22dfb89d5099ae90727fd863bc2af8c9 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Tue, 1 Nov 2022 20:52:54 +0800 Subject: [PATCH] Add support for hns DSCP Support DSCP for hns RoCE. Signed-off-by: Chengchang Tang --- 0011-Update-kernel-headers.patch | 38 ++++++++ 0012-libhns-Support-DSCP.patch | 150 +++++++++++++++++++++++++++++++ rdma-core.spec | 10 ++- 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 0011-Update-kernel-headers.patch create mode 100644 0012-libhns-Support-DSCP.patch diff --git a/0011-Update-kernel-headers.patch b/0011-Update-kernel-headers.patch new file mode 100644 index 0000000..7a86e19 --- /dev/null +++ b/0011-Update-kernel-headers.patch @@ -0,0 +1,38 @@ +From 12d2a17d404e3d5ba76863f64307ea52a7d15d15 Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Sat, 29 Oct 2022 10:44:17 +0800 +Subject: [PATCH 11/12] Update kernel headers + +To commit ?? ("RDMA/hns: Support DSCP of userspace"). + +Signed-off-by: Yixing Liu +--- + kernel-headers/rdma/hns-abi.h | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index 41738b8..542be5e 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -85,6 +85,18 @@ struct hns_roce_ib_create_qp_resp { + __aligned_u64 dwqe_mmap_key; + }; + ++struct hns_roce_ib_create_ah_resp { ++ __u8 priority; ++ __u8 tc_mode; ++ __u8 reserved[6]; ++}; ++ ++struct hns_roce_ib_modify_qp_resp { ++ __u8 tc_mode; ++ __u8 priority; ++ __u8 reserved[6]; ++}; ++ + enum { + HNS_ROCE_EXSGE_FLAGS = 1 << 0, + HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, +-- +2.30.0 + diff --git a/0012-libhns-Support-DSCP.patch b/0012-libhns-Support-DSCP.patch new file mode 100644 index 0000000..6433809 --- /dev/null +++ b/0012-libhns-Support-DSCP.patch @@ -0,0 +1,150 @@ +From b88e6ae3e144651092bce923123ca20361cdacab Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Tue, 27 Sep 2022 19:06:00 +0800 +Subject: [PATCH 12/12] libhns: Support DSCP + +This patch adds user mode DSCP function through +the mapping of dscp-tc configured in kernel mode. + +Signed-off-by: Yixing Liu +--- + providers/hns/hns_roce_u.h | 7 +++++++ + providers/hns/hns_roce_u_abi.h | 6 ++++++ + providers/hns/hns_roce_u_hw_v2.c | 19 +++++++++++++++---- + providers/hns/hns_roce_u_verbs.c | 7 +++++-- + 4 files changed, 33 insertions(+), 6 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 6b64cd0..8c1cb1e 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -175,6 +175,11 @@ enum hns_roce_db_type { + HNS_ROCE_DB_TYPE_NUM + }; + ++enum hns_roce_tc_map_mode { ++ HNS_ROCE_TC_MAP_MODE_PRIO, ++ HNS_ROCE_TC_MAP_MODE_DSCP, ++}; ++ + struct hns_roce_db_page { + struct hns_roce_db_page *prev, *next; + struct hns_roce_buf buf; +@@ -315,6 +320,8 @@ struct hns_roce_qp { + unsigned int next_sge; + int port_num; + uint8_t sl; ++ uint8_t tc_mode; ++ uint8_t priority; + unsigned int qkey; + enum ibv_mtu path_mtu; + +diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h +index 2753d30..0519ac7 100644 +--- a/providers/hns/hns_roce_u_abi.h ++++ b/providers/hns/hns_roce_u_abi.h +@@ -49,6 +49,9 @@ DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, + DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, + hns_roce_ib_alloc_ucontext, hns_roce_ib_alloc_ucontext_resp); + ++DECLARE_DRV_CMD(hns_roce_create_ah, IB_USER_VERBS_CMD_CREATE_AH, empty, ++ hns_roce_ib_create_ah_resp); ++ + DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP, + hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp); + +@@ -61,4 +64,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ, + DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ, + hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp); + ++DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, ++ empty, hns_roce_ib_modify_qp_resp); ++ + #endif /* _HNS_ROCE_U_ABI_H */ +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index a30d461..c652eea 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1543,10 +1543,11 @@ static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr, + static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) + { +- int ret; +- struct ibv_modify_qp cmd; ++ struct hns_roce_modify_qp_ex_resp resp_ex = {}; ++ struct hns_roce_modify_qp_ex cmd_ex = {}; + struct hns_roce_qp *hr_qp = to_hr_qp(qp); + bool flag = false; /* modify qp to error */ ++ int ret; + + if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { + pthread_spin_lock(&hr_qp->sq.lock); +@@ -1554,7 +1555,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + flag = true; + } + +- ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd)); ++ ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp_ex.ibv_resp, ++ sizeof(resp_ex)); + + if (flag) { + pthread_spin_unlock(&hr_qp->rq.lock); +@@ -1564,8 +1567,13 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + if (ret) + return ret; + +- if (attr_mask & IBV_QP_STATE) ++ if (attr_mask & IBV_QP_STATE) { + qp->state = attr->qp_state; ++ if (attr->qp_state == IBV_QPS_RTR) { ++ hr_qp->tc_mode = resp_ex.drv_payload.tc_mode; ++ hr_qp->priority = resp_ex.drv_payload.priority; ++ } ++ } + + if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) { + if (qp->recv_cq) +@@ -1579,6 +1587,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + hns_roce_init_qp_indices(to_hr_qp(qp)); + } + ++ if (hr_qp->tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) ++ hr_qp->sl = hr_qp->priority; ++ + record_qp_attr(qp, attr, attr_mask); + + return ret; +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index cff9d1d..3b7a67d 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1449,7 +1449,7 @@ static int get_tclass(struct ibv_context *context, struct ibv_ah_attr *attr, + struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) + { + struct hns_roce_device *hr_dev = to_hr_dev(pd->context->device); +- struct ib_uverbs_create_ah_resp resp = {}; ++ struct hns_roce_create_ah_resp resp = {}; + struct hns_roce_ah *ah; + + /* HIP08 don't support create ah */ +@@ -1477,12 +1477,15 @@ struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) + memcpy(ah->av.dgid, attr->grh.dgid.raw, ARRAY_SIZE(ah->av.dgid)); + } + +- if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp, sizeof(resp))) ++ if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp.ibv_resp, sizeof(resp))) + goto err; + + if (ibv_resolve_eth_l2_from_gid(pd->context, attr, ah->av.mac, NULL)) + goto err; + ++ if (resp.tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) ++ ah->av.sl = resp.priority; ++ + ah->av.udp_sport = get_ah_udp_sport(attr); + + return &ah->ibv_ah; +-- +2.30.0 + diff --git a/rdma-core.spec b/rdma-core.spec index b9112a5..9f1c3f5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 2 +Release: 3 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -16,6 +16,8 @@ Patch6: 0007-libhns-Refactor-rq-inline.patch Patch7: 0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch Patch8: 0009-Update-kernel-headers.patch Patch9: 0010-libhns-Support-cqe-inline.patch +Patch10: 0011-Update-kernel-headers.patch +Patch11: 0012-libhns-Support-DSCP.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -260,6 +262,12 @@ fi %{_mandir}/* %changelog +* Sat Oct 29 2022 tangchengchang - 41.0-3 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Support rq inline and cqe inline + * Sat Oct 08 2022 luoyouming - 41.0-2 - Type: requirement - ID: NA