From 5bcdc98f579f8fa2699318b3400f18ee9d629936 Mon Sep 17 00:00:00 2001 From: wuchangsheng Date: Tue, 30 Mar 2021 16:29:41 +0800 Subject: [PATCH] dpdk-support-gazelle-06-memalloc --- lib/librte_eal/linux/eal/eal_memalloc.c | 127 ++++++++++++++++++++---- 1 file changed, 110 insertions(+), 17 deletions(-) diff --git a/lib/librte_eal/linux/eal/eal_memalloc.c b/lib/librte_eal/linux/eal/eal_memalloc.c index cad4934..8e7f120 100644 --- a/lib/librte_eal/linux/eal/eal_memalloc.c +++ b/lib/librte_eal/linux/eal/eal_memalloc.c @@ -95,12 +95,14 @@ static int fallocate_supported = -1; /* unknown */ * they will be initialized at startup, and filled as we allocate/deallocate * segments. */ -static struct { +struct fd_list{ int *fds; /**< dynamically allocated array of segment lock fd's */ int memseg_list_fd; /**< memseg list fd */ int len; /**< total length of the array */ int count; /**< entries used in an array */ -} fd_list[RTE_MAX_MEMSEG_LISTS]; +}; +static struct fd_list fd_list[RTE_MAX_MEMSEG_LISTS]; +static struct fd_list sec_fd_list[RTE_MAX_SECONDARY][RTE_MAX_MEMSEG_LISTS]; /** local copy of a memory map, used to synchronize memory hotplug in MP */ static struct rte_memseg_list local_memsegs[RTE_MAX_MEMSEG_LISTS]; @@ -1391,13 +1393,13 @@ secondary_msl_create_walk(const struct rte_memseg_list *msl, } static int -alloc_list(int list_idx, int len) +__alloc_list(int list_idx, int len, struct fd_list *fd_ls) { int *data; int i; /* single-file segments mode does not need fd list */ - if (!internal_config.single_file_segments) { + if (!internal_config.single_file_segments) { // sec todo /* ensure we have space to store fd per each possible segment */ data = malloc(sizeof(int) * len); if (data == NULL) { @@ -1407,19 +1409,31 @@ alloc_list(int list_idx, int len) /* set all fd's as invalid */ for (i = 0; i < len; i++) data[i] = -1; - fd_list[list_idx].fds = data; - fd_list[list_idx].len = len; + fd_ls[list_idx].fds = data; + fd_ls[list_idx].len = len; } else { - fd_list[list_idx].fds = NULL; - fd_list[list_idx].len = 0; + fd_ls[list_idx].fds = NULL; + fd_ls[list_idx].len = 0; } - fd_list[list_idx].count = 0; - fd_list[list_idx].memseg_list_fd = -1; + fd_ls[list_idx].count = 0; + fd_ls[list_idx].memseg_list_fd = -1; return 0; } +static int +alloc_list(int list_idx, int len) +{ + return __alloc_list(list_idx, len, fd_list); +} + +static int +sec_alloc_list(int list_idx, int len, struct fd_list *fd_ls) +{ + return __alloc_list(list_idx, len, fd_ls); +} + static int fd_list_create_walk(const struct rte_memseg_list *msl, void *arg __rte_unused) @@ -1437,27 +1451,71 @@ fd_list_create_walk(const struct rte_memseg_list *msl, return alloc_list(msl_idx, len); } -int -eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd) +static int +fd_list_destroy_walk(const struct rte_memseg_list *msl, const int sec_idx) { - struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; + struct rte_mem_config *mcfg = rte_eal_sec_get_configuration(sec_idx)->mem_config; + struct fd_list *fd_ls = sec_fd_list[sec_idx]; + int list_idx; + + list_idx = msl - mcfg->memsegs; + if (fd_ls[list_idx].len != 0) { + free(fd_ls[list_idx].fds); + /* We have closed fd, seeing in function of eal_legacy_hugepage_attach. */ + //close(fd_ls[list_idx].fds[seg_idx]); + } + memset(&fd_ls[list_idx], 0, sizeof(fd_ls[list_idx])); + + return 0; +} + +static int +__eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd, + const struct rte_config *rte_cfg, struct fd_list *fd_ls) +{ + struct rte_mem_config *mcfg = rte_cfg->mem_config; /* single file segments mode doesn't support individual segment fd's */ - if (internal_config.single_file_segments) + if (internal_config.single_file_segments) // sec todo return -ENOTSUP; /* if list is not allocated, allocate it */ - if (fd_list[list_idx].len == 0) { + if (fd_ls[list_idx].len == 0) { int len = mcfg->memsegs[list_idx].memseg_arr.len; - if (alloc_list(list_idx, len) < 0) + if (sec_alloc_list(list_idx, len, fd_ls) < 0) return -ENOMEM; } - fd_list[list_idx].fds[seg_idx] = fd; + fd_ls[list_idx].fds[seg_idx] = fd; return 0; } +int +eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd) +{ + return __eal_memalloc_set_seg_fd(list_idx, seg_idx, fd, + rte_eal_get_configuration(), fd_list); +} + +int +eal_sec_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd, + const int switch_pri_and_sec, const int sec_idx) +{ + struct rte_config *rte_cfg = NULL; + struct fd_list *fd_ls = NULL; + + if (!switch_pri_and_sec) { + rte_cfg = rte_eal_get_configuration(); + fd_ls = &fd_list[0]; + } else { + rte_cfg = rte_eal_sec_get_configuration(sec_idx); + fd_ls = &sec_fd_list[sec_idx][0]; + } + + return __eal_memalloc_set_seg_fd(list_idx, seg_idx, fd, rte_cfg, fd_ls); +} + int eal_memalloc_set_seg_list_fd(int list_idx, int fd) { @@ -1602,3 +1660,38 @@ eal_memalloc_init(void) return -1; return 0; } + +int +eal_memalloc_destroy(const int sec_idx) +{ + int msl_idx = 0; + struct rte_memseg_list *msl; + struct rte_mem_config *mcfg = rte_eal_sec_get_configuration(sec_idx)->mem_config; + + for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) { + + msl = &mcfg->memsegs[msl_idx]; + + /* skip empty memseg lists */ + if (msl->memseg_arr.len == 0) + continue; + + if (rte_sec_fbarray_destroy(&msl->memseg_arr, sec_idx)) { + RTE_LOG(ERR, EAL, "Cannot clear secondary process local memseg lists\n"); + return -1; + } + + if (munmap(msl->base_va, msl->len) < 0) { + RTE_LOG(ERR, EAL, "Failed to unmap memseg lists\n"); + return -1; + } + memset(msl, 0, sizeof(*msl)); + + if (fd_list_destroy_walk(msl, sec_idx)) { + RTE_LOG(ERR, EAL, "Failed to clear secondary fd_list.\n"); + return -1; + } + } + + return 0; +} -- 2.23.0