MFC r264346 (by alc):

Pass MAP_ALIGNED_SUPER to allocate the whole dso region if its text is large
enough for the superpage mapping.

4 files changed, 67 insertions, 25 deletions

diff --git a/libexec/rtld-elf/malloc.c b/libexec/rtld-elf/malloc.c
index 8e34362..9f7dbe0 100644
--- a/libexec/rtld-elf/malloc.c
+++ b/libexec/rtld-elf/malloc.c
@@ -139,25 +139,14 @@ botch(s)
 /* Debugging stuff */
 #define TRACE()	rtld_printf("TRACE %s:%d\n", __FILE__, __LINE__)
 
-extern int pagesize;
-
-static int
-rtld_getpagesize(void)
-{
-	int mib[2];
-	size_t size;
-
-	if (pagesize != 0)
-		return (pagesize);
-
-	mib[0] = CTL_HW;
-	mib[1] = HW_PAGESIZE;
-	size = sizeof(pagesize);
-	if (sysctl(mib, 2, &pagesize, &size, NULL, 0) == -1)
-		return (-1);
-	return (pagesize);
-
-}
+/*
+ * The array of supported page sizes is provided by the user, i.e., the
+ * program that calls this storage allocator.  That program must initialize
+ * the array before making its first call to allocate storage.  The array
+ * must contain at least one page size.  The page sizes must be stored in
+ * increasing order.
+ */
+extern size_t *pagesizes;
 
 void *
 malloc(nbytes)
@@ -173,7 +162,7 @@ malloc(nbytes)
 	 * align break pointer so all data will be page aligned.
 	 */
 	if (pagesz == 0) {
-		pagesz = n = rtld_getpagesize();
+		pagesz = n = pagesizes[0];
 		if (morepages(NPOOLPAGES) == 0)
 			return NULL;
 		op = (union overhead *)(pagepool_start);
diff --git a/libexec/rtld-elf/map_object.c b/libexec/rtld-elf/map_object.c
index 0f75cca..2e17fbf 100644
--- a/libexec/rtld-elf/map_object.c
+++ b/libexec/rtld-elf/map_object.c
@@ -68,6 +68,7 @@ map_object(int fd, const char *path, const struct stat *sb)
     Elf_Addr base_vaddr;
     Elf_Addr base_vlimit;
     caddr_t base_addr;
+    int base_flags;
     Elf_Off data_offset;
     Elf_Addr data_vaddr;
     Elf_Addr data_vlimit;
@@ -176,9 +177,11 @@ map_object(int fd, const char *path, const struct stat *sb)
     base_vlimit = round_page(segs[nsegs]->p_vaddr + segs[nsegs]->p_memsz);
     mapsize = base_vlimit - base_vaddr;
     base_addr = (caddr_t) base_vaddr;
+    base_flags = MAP_PRIVATE | MAP_ANON | MAP_NOCORE;
+    if (npagesizes > 1 && round_page(segs[0]->p_filesz) >= pagesizes[1])
+	base_flags |= MAP_ALIGNED_SUPER;
 
-    mapbase = mmap(base_addr, mapsize, PROT_NONE, MAP_ANON | MAP_PRIVATE |
-      MAP_NOCORE, -1, 0);
+    mapbase = mmap(base_addr, mapsize, PROT_NONE, base_flags, -1, 0);
     if (mapbase == (caddr_t) -1) {
 	_rtld_error("%s: mmap of entire address space failed: %s",
 	  path, rtld_strerror(errno));
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index f5a6a17..4753d13 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -97,6 +97,7 @@ static void *fill_search_info(const char *, size_t, void *);
 static char *find_library(const char *, const Obj_Entry *);
 static const char *gethints(bool);
 static void init_dag(Obj_Entry *);
+static void init_pagesizes(Elf_Auxinfo **aux_info);
 static void init_rtld(caddr_t, Elf_Auxinfo **);
 static void initlist_add_neededs(Needed_Entry *, Objlist *);
 static void initlist_add_objects(Obj_Entry *, Obj_Entry **, Objlist *);
@@ -206,7 +207,8 @@ extern Elf_Dyn _DYNAMIC;
 #define	RTLD_IS_DYNAMIC()	(&_DYNAMIC != NULL)
 #endif
 
-int osreldate, pagesize;
+int npagesizes, osreldate;
+size_t *pagesizes;
 
 long __stack_chk_guard[8] = {0, 0, 0, 0, 0, 0, 0, 0};
 
@@ -1824,8 +1826,9 @@ init_rtld(caddr_t mapbase, Elf_Auxinfo **aux_info)
     /* Now that non-local variables can be accesses, copy out obj_rtld. */
     memcpy(&obj_rtld, &objtmp, sizeof(obj_rtld));
 
-    if (aux_info[AT_PAGESZ] != NULL)
-	    pagesize = aux_info[AT_PAGESZ]->a_un.a_val;
+    /* The page size is required by the dynamic memory allocator. */
+    init_pagesizes(aux_info);
+
     if (aux_info[AT_OSRELDATE] != NULL)
 	    osreldate = aux_info[AT_OSRELDATE]->a_un.a_val;
 
@@ -1839,6 +1842,50 @@ init_rtld(caddr_t mapbase, Elf_Auxinfo **aux_info)
 }
 
 /*
+ * Retrieve the array of supported page sizes.  The kernel provides the page
+ * sizes in increasing order.
+ */
+static void
+init_pagesizes(Elf_Auxinfo **aux_info)
+{
+	static size_t psa[MAXPAGESIZES];
+	int mib[2];
+	size_t len, size;
+
+	if (aux_info[AT_PAGESIZES] != NULL && aux_info[AT_PAGESIZESLEN] !=
+	    NULL) {
+		size = aux_info[AT_PAGESIZESLEN]->a_un.a_val;
+		pagesizes = aux_info[AT_PAGESIZES]->a_un.a_ptr;
+	} else {
+		len = 2;
+		if (sysctlnametomib("hw.pagesizes", mib, &len) == 0)
+			size = sizeof(psa);
+		else {
+			/* As a fallback, retrieve the base page size. */
+			size = sizeof(psa[0]);
+			if (aux_info[AT_PAGESZ] != NULL) {
+				psa[0] = aux_info[AT_PAGESZ]->a_un.a_val;
+				goto psa_filled;
+			} else {
+				mib[0] = CTL_HW;
+				mib[1] = HW_PAGESIZE;
+				len = 2;
+			}
+		}
+		if (sysctl(mib, len, psa, &size, NULL, 0) == -1) {
+			_rtld_error("sysctl for hw.pagesize(s) failed");
+			die();
+		}
+psa_filled:
+		pagesizes = psa;
+	}
+	npagesizes = size / sizeof(pagesizes[0]);
+	/* Discard any invalid entries at the end of the array. */
+	while (npagesizes > 0 && pagesizes[npagesizes - 1] == 0)
+		npagesizes--;
+}
+
+/*
  * Add the init functions from a needed object list (and its recursive
  * needed objects) to "list".  This is not used directly; it is a helper
  * function for initlist_add_objects().  The write lock must be held
diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h
index a1d515b..e539117 100644
--- a/libexec/rtld-elf/rtld.h
+++ b/libexec/rtld-elf/rtld.h
@@ -71,6 +71,9 @@ extern size_t tls_static_space;
 extern int tls_dtv_generation;
 extern int tls_max_index;
 
+extern int npagesizes;
+extern size_t *pagesizes;
+
 extern int main_argc;
 extern char **main_argv;
 extern char **environ;