780caed580
backport the following seven patches malloc: Add madvise support for Transparent Huge Pages malloc: Add THP/madvise support for sbrk malloc: Move mmap logic to its own function malloc: Add Huge Page support for mmap malloc: Add Huge Page support to arenas malloc: Move MORECORE fallback mmap to sysmalloc_mmap_fallback malloc: Enable huge page support on main arena Signed-off-by: Yang Yanchao <yangyanchao6@huawei.com>
112 lines
3.6 KiB
Diff
112 lines
3.6 KiB
Diff
From 7478c9959ae409f7b3d63146943575d6ee745352 Mon Sep 17 00:00:00 2001
|
|
From: Adhemerval Zanella <adhemerval.zanella@linaro.org>
|
|
Date: Fri, 13 Aug 2021 10:06:04 -0300
|
|
Subject: [PATCH 2/7] malloc: Add THP/madvise support for sbrk
|
|
|
|
To increase effectiveness with Transparent Huge Page with madvise, the
|
|
large page size is use instead page size for sbrk increment for the
|
|
main arena.
|
|
|
|
Checked on x86_64-linux-gnu.
|
|
|
|
Reviewed-by: DJ Delorie <dj@redhat.com>
|
|
---
|
|
include/libc-pointer-arith.h | 8 ++++++++
|
|
malloc/malloc.c | 34 +++++++++++++++++++++++++++++-----
|
|
2 files changed, 37 insertions(+), 5 deletions(-)
|
|
|
|
diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
|
|
index 04ba537617..55dccc10ac 100644
|
|
--- a/include/libc-pointer-arith.h
|
|
+++ b/include/libc-pointer-arith.h
|
|
@@ -60,4 +60,12 @@
|
|
#define PTR_ALIGN_UP(base, size) \
|
|
((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
|
|
|
|
+/* Check if BASE is aligned on SIZE */
|
|
+#define PTR_IS_ALIGNED(base, size) \
|
|
+ ((((uintptr_t) (base)) & (size - 1)) == 0)
|
|
+
|
|
+/* Returns the ptrdiff_t diference between P1 and P2. */
|
|
+#define PTR_DIFF(p1, p2) \
|
|
+ ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
|
|
+
|
|
#endif
|
|
diff --git a/malloc/malloc.c b/malloc/malloc.c
|
|
index c75841b841..57db4dd9a5 100644
|
|
--- a/malloc/malloc.c
|
|
+++ b/malloc/malloc.c
|
|
@@ -2023,6 +2023,16 @@ madvise_thp (void *p, INTERNAL_SIZE_T size)
|
|
not active. */
|
|
if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
|
|
return;
|
|
+
|
|
+ /* Linux requires the input address to be page-aligned, and unaligned
|
|
+ inputs happens only for initial data segment. */
|
|
+ if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
|
|
+ {
|
|
+ void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
|
|
+ size += PTR_DIFF (p, q);
|
|
+ p = q;
|
|
+ }
|
|
+
|
|
__madvise (p, size, MADV_HUGEPAGE);
|
|
#endif
|
|
}
|
|
@@ -2609,14 +2619,25 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
|
|
size -= old_size;
|
|
|
|
/*
|
|
- Round to a multiple of page size.
|
|
+ Round to a multiple of page size or huge page size.
|
|
If MORECORE is not contiguous, this ensures that we only call it
|
|
with whole-page arguments. And if MORECORE is contiguous and
|
|
this is not first time through, this preserves page-alignment of
|
|
previous calls. Otherwise, we correct to page-align below.
|
|
*/
|
|
|
|
- size = ALIGN_UP (size, pagesize);
|
|
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
|
|
+ /* Defined in brk.c. */
|
|
+ extern void *__curbrk;
|
|
+ if (__glibc_unlikely (mp_.thp_pagesize != 0))
|
|
+ {
|
|
+ uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
|
|
+ mp_.thp_pagesize);
|
|
+ size = top - (uintptr_t) __curbrk;
|
|
+ }
|
|
+ else
|
|
+#endif
|
|
+ size = ALIGN_UP (size, GLRO(dl_pagesize));
|
|
|
|
/*
|
|
Don't try to call MORECORE if argument is so big as to appear
|
|
@@ -2899,10 +2920,8 @@ systrim (size_t pad, mstate av)
|
|
long released; /* Amount actually released */
|
|
char *current_brk; /* address returned by pre-check sbrk call */
|
|
char *new_brk; /* address returned by post-check sbrk call */
|
|
- size_t pagesize;
|
|
long top_area;
|
|
|
|
- pagesize = GLRO (dl_pagesize);
|
|
top_size = chunksize (av->top);
|
|
|
|
top_area = top_size - MINSIZE - 1;
|
|
@@ -2910,7 +2929,12 @@ systrim (size_t pad, mstate av)
|
|
return 0;
|
|
|
|
/* Release in pagesize units and round down to the nearest page. */
|
|
- extra = ALIGN_DOWN(top_area - pad, pagesize);
|
|
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
|
|
+ if (__glibc_unlikely (mp_.thp_pagesize != 0))
|
|
+ extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
|
|
+ else
|
|
+#endif
|
|
+ extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
|
|
|
|
if (extra == 0)
|
|
return 0;
|
|
--
|
|
2.33.0
|
|
|