Bug 683597 - Change jemalloc's accounting for huge allocations to round up to the next page boundary, not to the next MB. r=khuey
authorJustin Lebar <justin.lebar@gmail.com>
Fri, 07 Oct 2011 14:39:53 -0400
changeset 78491 21df455d50831fd2007d2b4745f1ad2395b97df9
parent 78490 8692821e87e1f370c5bfd9df61023605fcd6a67b
child 78492 58c9046d8485f1dac0b9b678185ad175d3a0f7fe
push id2544
push userjlebar@mozilla.com
push dateMon, 10 Oct 2011 15:53:29 +0000
treeherdermozilla-inbound@21df455d5083 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskhuey
bugs683597
milestone10.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 683597 - Change jemalloc's accounting for huge allocations to round up to the next page boundary, not to the next MB. r=khuey
memory/jemalloc/jemalloc.c
--- a/memory/jemalloc/jemalloc.c
+++ b/memory/jemalloc/jemalloc.c
@@ -4674,19 +4674,17 @@ arenas_extend(unsigned ind)
  * Begin general internal functions.
  */
 
 static void *
 huge_malloc(size_t size, bool zero)
 {
 	void *ret;
 	size_t csize;
-#ifdef MALLOC_DECOMMIT
 	size_t psize;
-#endif
 	extent_node_t *node;
 
 	/* Allocate one or more contiguous chunks for this request. */
 
 	csize = CHUNK_CEILING(size);
 	if (csize == 0) {
 		/* size is large enough to cause size_t wrap-around. */
 		return (NULL);
@@ -4700,32 +4698,43 @@ huge_malloc(size_t size, bool zero)
 	ret = chunk_alloc(csize, zero, true);
 	if (ret == NULL) {
 		base_node_dealloc(node);
 		return (NULL);
 	}
 
 	/* Insert node into huge. */
 	node->addr = ret;
-#ifdef MALLOC_DECOMMIT
 	psize = PAGE_CEILING(size);
 	node->size = psize;
-#else
-	node->size = csize;
-#endif
 
 	malloc_mutex_lock(&huge_mtx);
 	extent_tree_ad_insert(&huge, node);
 #ifdef MALLOC_STATS
 	huge_nmalloc++;
-#  ifdef MALLOC_DECOMMIT
+
+        /* Although we allocated space for csize bytes, we indicate that we've
+         * allocated only psize bytes.
+         *
+         * If DECOMMIT is defined, this is a reasonable thing to do, since
+         * we'll explicitly decommit the bytes in excess of psize.
+         *
+         * If DECOMMIT is not defined, then we're relying on the OS to be lazy
+         * about how it allocates physical pages to mappings.  If we never
+         * touch the pages in excess of psize, the OS won't allocate a physical
+         * page, and we won't use more than psize bytes of physical memory.
+         *
+         * A correct program will only touch memory in excess of how much it
+         * requested if it first calls malloc_usable_size and finds out how
+         * much space it has to play with.  But because we set node->size =
+         * psize above, malloc_usable_size will return psize, not csize, and
+         * the program will (hopefully) never touch bytes in excess of psize.
+         * Thus those bytes won't take up space in physical memory, and we can
+         * reasonably claim we never "allocated" them in the first place. */
 	huge_allocated += psize;
-#  else
-	huge_allocated += csize;
-#  endif
 #endif
 	malloc_mutex_unlock(&huge_mtx);
 
 #ifdef MALLOC_DECOMMIT
 	if (csize - psize > 0)
 		pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
 #endif
 
@@ -4756,19 +4765,17 @@ huge_malloc(size_t size, bool zero)
 }
 
 /* Only handles large allocations that require more than chunk alignment. */
 static void *
 huge_palloc(size_t alignment, size_t size)
 {
 	void *ret;
 	size_t alloc_size, chunk_size, offset;
-#ifdef MALLOC_DECOMMIT
 	size_t psize;
-#endif
 	extent_node_t *node;
 	int pfd;
 
 	/*
 	 * This allocation requires alignment that is even larger than chunk
 	 * alignment.  This means that huge_malloc() isn't good enough.
 	 *
 	 * Allocate almost twice as many chunks as are demanded by the size or
@@ -4827,32 +4834,26 @@ huge_palloc(size_t alignment, size_t siz
 		/*
 		 * Failure here indicates a race with another thread, so try
 		 * again.
 		 */
 	} while (ret == NULL);
 #endif
 	/* Insert node into huge. */
 	node->addr = ret;
-#ifdef MALLOC_DECOMMIT
 	psize = PAGE_CEILING(size);
 	node->size = psize;
-#else
-	node->size = chunk_size;
-#endif
 
 	malloc_mutex_lock(&huge_mtx);
 	extent_tree_ad_insert(&huge, node);
 #ifdef MALLOC_STATS
 	huge_nmalloc++;
-#  ifdef MALLOC_DECOMMIT
+        /* See note in huge_alloc() for why huge_allocated += psize is correct
+         * here even when DECOMMIT is not defined. */
 	huge_allocated += psize;
-#  else
-	huge_allocated += chunk_size;
-#  endif
 #endif
 	malloc_mutex_unlock(&huge_mtx);
 
 #ifdef MALLOC_DECOMMIT
 	if (chunk_size - psize > 0) {
 		pages_decommit((void *)((uintptr_t)ret + psize),
 		    chunk_size - psize);
 	}
@@ -4892,19 +4893,17 @@ huge_ralloc(void *ptr, size_t size, size
 {
 	void *ret;
 	size_t copysize;
 
 	/* Avoid moving the allocation if the size class would not change. */
 
 	if (oldsize > arena_maxclass &&
 	    CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
-#ifdef MALLOC_DECOMMIT
 		size_t psize = PAGE_CEILING(size);
-#endif
 #ifdef MALLOC_FILL
 		if (opt_junk && size < oldsize) {
 			memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
 			    - size);
 		}
 #endif
 #ifdef MALLOC_DECOMMIT
 		if (psize < oldsize) {
@@ -4920,34 +4919,42 @@ huge_ralloc(void *ptr, size_t size, size
 			assert(node != NULL);
 			assert(node->size == oldsize);
 #  ifdef MALLOC_STATS
 			huge_allocated -= oldsize - psize;
 #  endif
 			node->size = psize;
 			malloc_mutex_unlock(&huge_mtx);
 		} else if (psize > oldsize) {
-			extent_node_t *node, key;
-
 			pages_commit((void *)((uintptr_t)ptr + oldsize),
 			    psize - oldsize);
-
-			/* Update recorded size. */
-			malloc_mutex_lock(&huge_mtx);
-			key.addr = __DECONST(void *, ptr);
-			node = extent_tree_ad_search(&huge, &key);
-			assert(node != NULL);
-			assert(node->size == oldsize);
+                }
+#endif
+
+                /* Although we don't have to commit or decommit anything if
+                 * DECOMMIT is not defined and the size class didn't change, we
+                 * do need to update the recorded size if the size increased,
+                 * so malloc_usable_size doesn't return a value smaller than
+                 * what was requested via realloc(). */
+
+                if (psize > oldsize) {
+                        /* Update recorded size. */
+                        extent_node_t *node, key;
+                        malloc_mutex_lock(&huge_mtx);
+                        key.addr = __DECONST(void *, ptr);
+                        node = extent_tree_ad_search(&huge, &key);
+                        assert(node != NULL);
+                        assert(node->size == oldsize);
 #  ifdef MALLOC_STATS
-			huge_allocated += psize - oldsize;
+                        huge_allocated += psize - oldsize;
 #  endif
-			node->size = psize;
-			malloc_mutex_unlock(&huge_mtx);
-		}
-#endif
+                        node->size = psize;
+                        malloc_mutex_unlock(&huge_mtx);
+                }
+
 #ifdef MALLOC_FILL
 		if (opt_zero && size > oldsize) {
 			memset((void *)((uintptr_t)ptr + oldsize), 0, size
 			    - oldsize);
 		}
 #endif
 		return (ptr);
 	}
@@ -4993,21 +5000,17 @@ huge_dalloc(void *ptr)
 
 	malloc_mutex_unlock(&huge_mtx);
 
 	/* Unmap chunk. */
 #ifdef MALLOC_FILL
 	if (opt_junk)
 		memset(node->addr, 0x5a, node->size);
 #endif
-#ifdef MALLOC_DECOMMIT
 	chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
-#else
-	chunk_dealloc(node->addr, node->size);
-#endif
 	VALGRIND_FREELIKE_BLOCK(node->addr, 0);
 
 	base_node_dealloc(node);
 }
 
 #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
 #ifdef MOZ_MEMORY_BSD
 static inline unsigned