Bug 422960: Add jemalloc_stats() and jemalloc.h, r=benjamin
--- a/js/src/Makefile.in
+++ b/js/src/Makefile.in
@@ -354,16 +354,22 @@ endif # JS_ULTRASPARC_OPTS
endif
ifeq ($(OS_RELEASE),4.1)
LDFLAGS += -ldl -lnsl
else
LDFLAGS += -lposix4 -ldl -lnsl -lsocket
endif
endif
+ifdef MOZ_MEMORY
+ifeq ($(OS_ARCH),Darwin)
+LDFLAGS += -ljemalloc
+endif
+endif
+
# Allow building jsinterp.c with special optimization flags
ifdef INTERP_OPTIMIZER
jsinterp.$(OBJ_SUFFIX): MODULE_OPTIMIZE_FLAGS=$(INTERP_OPTIMIZER)
endif
ifeq ($(OS_ARCH),IRIX)
ifndef GNU_CC
_COMPILE_CFLAGS = $(patsubst -O%,-O1,$(COMPILE_CFLAGS))
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -83,24 +83,22 @@
*/
#if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 || MOZ_MEMORY
# define HAS_POSIX_MEMALIGN 1
#else
# define HAS_POSIX_MEMALIGN 0
#endif
/*
- * jemalloc provides posix_memalign but the function has to be explicitly
- * declared on Windows.
+ * jemalloc provides posix_memalign.
*/
-#if HAS_POSIX_MEMALIGN && MOZ_MEMORY_WINDOWS
-JS_BEGIN_EXTERN_C
-extern int
-posix_memalign(void **memptr, size_t alignment, size_t size);
-JS_END_EXTERN_C
+#ifdef MOZ_MEMORY
+extern "C" {
+#include "../../memory/jemalloc/jemalloc.h"
+}
#endif
/*
* Include the headers for mmap unless we have posix_memalign and do not
* insist on mmap.
*/
#if JS_GC_USE_MMAP || (!defined JS_GC_USE_MMAP && !HAS_POSIX_MEMALIGN)
# if defined(XP_WIN)
--- a/memory/jemalloc/Makefile.in
+++ b/memory/jemalloc/Makefile.in
@@ -40,16 +40,20 @@ DEPTH = ../..
topsrcdir = @top_srcdir@
srcdir = @srcdir@
VPATH = @srcdir@
include $(DEPTH)/config/autoconf.mk
MODULE = jemalloc
+# jemalloc.c properly uses 'static', so don't burden it with manually exposing
+# symbols.
+VISIBILITY_FLAGS=
+
ifeq (WINNT,$(OS_TARGET))
# Two options for Windows, either you build the CRT from source,
# or you use a pre-built DLL.
ifneq (,$(WIN32_CRT_SRC_DIR))
# Building the CRT from source
CRT_OBJ_DIR=./$(shell basename "$(WIN32_CRT_SRC_DIR)")
libs:: $(CRT_OBJ_DIR)/build/intel/mozcrt19.dll
$(INSTALL) $< $(FINAL_TARGET)
@@ -63,18 +67,19 @@ libs:: $(CRT_OBJ_DIR)/build/intel/mozcrt
pushd $(CRT_OBJ_DIR)/intel/$${i}_lib && lib -extract:..\\build\\intel\\$${i}_obj\\unhandld.obj eh.lib && popd; \
done
# truly awful
#XXX: get ed into mozillabuild, bug 415123
$(PERL) $(srcdir)/apply-ed-patches.pl $(srcdir)/crtsp1.diff \
$(CRT_OBJ_DIR) $(srcdir)/ed.exe
$(CRT_OBJ_DIR)/build/intel/mozcrt19.dll: \
- $(CRT_OBJ_DIR)/jemalloc.c $(srcdir)/jemalloc.c $(srcdir)/tree.h
- cp $(srcdir)/jemalloc.c $(srcdir)/tree.h $(CRT_OBJ_DIR)
+ $(CRT_OBJ_DIR)/jemalloc.c $(srcdir)/jemalloc.c $(srcdir)/jemalloc.h \
+ $(srcdir)/tree.h
+ cp $(srcdir)/jemalloc.c $(srcdir)/jemalloc.h $(srcdir)/tree.h $(CRT_OBJ_DIR)
# this pretty much sucks, but nmake and make don't play well together
$(PYTHON) $(srcdir)/build-crt.py $(CRT_OBJ_DIR)
#XXX: these don't link right for some reason
rm $(CRT_OBJ_DIR)/build/intel/{libcmt,libcpmt}.lib
else
# Using a pre-built DLL, so just install it.
libs:: $(WIN32_CUSTOM_CRT_DIR)/mozcrt19.dll
$(INSTALL) $< $(FINAL_TARGET)
--- a/memory/jemalloc/jemalloc.c
+++ b/memory/jemalloc/jemalloc.c
@@ -98,26 +98,29 @@
* MALLOC_PRODUCTION disables assertions and statistics gathering. It also
* defaults the A and J runtime options to off. These settings are appropriate
* for production systems.
*/
#ifndef MOZ_MEMORY_DEBUG
# define MALLOC_PRODUCTION
#endif
+/*
+ * MALLOC_STATS enables statistics calculation, and is required for
+ * jemalloc_stats().
+ */
+#define MALLOC_STATS
+
#ifndef MALLOC_PRODUCTION
/*
* MALLOC_DEBUG enables assertions and other sanity checks, and disables
* inline functions.
*/
# define MALLOC_DEBUG
- /* MALLOC_STATS enables statistics calculation. */
-# define MALLOC_STATS
-
/* Memory filling (junk/zero). */
# define MALLOC_FILL
/* Allocation tracing. */
# define MALLOC_UTRACE
/* Support optional abort() on OOM. */
# define MALLOC_XMALLOC
@@ -198,17 +201,16 @@
#include <cruntime.h>
#include <internal.h>
#include <windows.h>
#include <io.h>
#include "tree.h"
#pragma warning( disable: 4267 4996 4146 )
-#define bool BOOL
#define false FALSE
#define true TRUE
#define inline __inline
#define SIZE_T_MAX SIZE_MAX
#define STDERR_FILENO 2
#define PATH_MAX MAX_PATH
#define vsnprintf _vsnprintf
#define assert(f) /* we can't assert in the CRT */
@@ -331,16 +333,18 @@ typedef unsigned long long uintmax_t;
#endif
#ifndef MOZ_MEMORY
#include "un-namespace.h"
#endif
#endif
+#include "jemalloc.h"
+
#ifdef MOZ_MEMORY_DARWIN
static const bool __isthreaded = true;
#endif
#define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
#ifdef MALLOC_DEBUG
# ifdef NDEBUG
@@ -359,22 +363,16 @@ static const bool __isthreaded = true;
/* Disable inlining to make debugging easier. */
#ifdef inline
#undef inline
#endif
# define inline
#endif
-#ifdef __GNUC__
-#define VISIBLE __attribute__((visibility("default")))
-#else
-#define VISIBLE
-#endif
-
/* Size of stack-allocated buffer passed to strerror_r(). */
#define STRERROR_BUF 64
/* Minimum alignment of allocations is 2^QUANTUM_2POW_MIN bytes. */
# define QUANTUM_2POW_MIN 4
#ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
# define SIZEOF_PTR_2POW MOZ_MEMORY_SIZEOF_PTR_2POW
#else
@@ -1114,17 +1112,17 @@ typedef struct {
static bool malloc_mutex_init(malloc_mutex_t *mutex);
static bool malloc_spin_init(malloc_spinlock_t *lock);
static void wrtmessage(const char *p1, const char *p2, const char *p3,
const char *p4);
#ifdef MALLOC_STATS
#ifdef MOZ_MEMORY_DARWIN
/* Avoid namespace collision with OS X's malloc APIs. */
-#define malloc_printf xmalloc_printf
+#define malloc_printf moz_malloc_printf
#endif
static void malloc_printf(const char *format, ...);
#endif
static char *umax2s(uintmax_t x, char *s);
#ifdef MALLOC_DSS
static bool base_pages_alloc_dss(size_t minsize);
#endif
static bool base_pages_alloc_mmap(size_t minsize);
@@ -5730,24 +5728,38 @@ malloc_shutdown()
/*
* End general internal functions.
*/
/******************************************************************************/
/*
* Begin malloc(3)-compatible functions.
*/
-VISIBLE
+/*
+ * Inline the standard malloc functions if they are being subsumed by Darwin's
+ * zone infrastructure.
+ */
#ifdef MOZ_MEMORY_DARWIN
-inline void *
-moz_malloc(size_t size)
+# define ZONE_INLINE inline
#else
+# define ZONE_INLINE
+#endif
+
+/* Mangle standard interfaces on Darwin, in order to avoid linking problems. */
+#ifdef MOZ_MEMORY_DARWIN
+#define malloc(a) moz_malloc(a)
+#define valloc(a) moz_valloc(a)
+#define calloc(a, b) moz_calloc(a, b)
+#define realloc(a, b) moz_realloc(a, b)
+#define free(a) moz_free(a)
+#endif
+
+ZONE_INLINE
void *
malloc(size_t size)
-#endif
{
void *ret;
if (malloc_init()) {
ret = NULL;
goto RETURN;
}
@@ -5778,36 +5790,29 @@ RETURN:
#endif
errno = ENOMEM;
}
UTRACE(0, size, ret);
return (ret);
}
-#ifdef MOZ_MEMORY_DARWIN
-VISIBLE
-inline void *
-moz_memalign(size_t alignment, size_t size)
-#elif (defined(MOZ_MEMORY_SOLARIS))
+#ifdef MOZ_MEMORY_SOLARIS
# ifdef __SUNPRO_C
void *
memalign(size_t alignment, size_t size);
#pragma no_inline(memalign)
-# elif (defined(__GNU_C__)
+# elif (defined(__GNU_C__))
__attribute__((noinline))
# endif
-VISIBLE
+#else
+inline
+#endif
void *
memalign(size_t alignment, size_t size)
-#else
-VISIBLE
-inline void *
-memalign(size_t alignment, size_t size)
-#endif
{
void *ret;
assert(((alignment - 1) & alignment) == 0 && alignment >=
sizeof(void *));
if (malloc_init()) {
ret = NULL;
@@ -5823,24 +5828,19 @@ RETURN:
": (malloc) Error in memalign(): out of memory\n", "", "");
abort();
}
#endif
UTRACE(0, size, ret);
return (ret);
}
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline int
-moz_posix_memalign(void **memptr, size_t alignment, size_t size)
-#else
+ZONE_INLINE
int
posix_memalign(void **memptr, size_t alignment, size_t size)
-#endif
{
void *result;
/* Make sure that alignment is a large enough power of 2. */
if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
#ifdef MALLOC_XMALLOC
if (opt_xmalloc) {
_malloc_message(_getprogname(),
@@ -5859,40 +5859,30 @@ posix_memalign(void **memptr, size_t ali
#endif
if (result == NULL)
return (ENOMEM);
*memptr = result;
return (0);
}
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline void *
-moz_valloc(size_t size)
-#else
+ZONE_INLINE
void *
valloc(size_t size)
-#endif
{
#ifdef MOZ_MEMORY_DARWIN
return (moz_memalign(pagesize, size));
#else
return (memalign(pagesize, size));
#endif
}
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline void *
-moz_calloc(size_t num, size_t size)
-#else
+ZONE_INLINE
void *
calloc(size_t num, size_t size)
-#endif
{
void *ret;
size_t num_size;
if (malloc_init()) {
num_size = 0;
ret = NULL;
goto RETURN;
@@ -5936,24 +5926,19 @@ RETURN:
#endif
errno = ENOMEM;
}
UTRACE(0, num_size, ret);
return (ret);
}
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline void *
-moz_realloc(void *ptr, size_t size)
-#else
+ZONE_INLINE
void *
realloc(void *ptr, size_t size)
-#endif
{
void *ret;
if (size == 0) {
#ifdef MALLOC_SYSV
if (opt_sysv == false)
#endif
size = 1;
@@ -6004,24 +5989,19 @@ realloc(void *ptr, size_t size)
#ifdef MALLOC_SYSV
RETURN:
#endif
UTRACE(ptr, size, ret);
return (ret);
}
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline void
-moz_free(void *ptr)
-#else
+ZONE_INLINE
void
free(void *ptr)
-#endif
{
UTRACE(ptr, 0, 0);
if (ptr != NULL) {
assert(malloc_initialized);
idalloc(ptr);
}
@@ -6030,35 +6010,144 @@ free(void *ptr)
/*
* End malloc(3)-compatible functions.
*/
/******************************************************************************/
/*
* Begin non-standard functions.
*/
-VISIBLE
-#ifdef MOZ_MEMORY_DARWIN
-inline size_t
-moz_malloc_usable_size(const void *ptr)
-#else
size_t
malloc_usable_size(const void *ptr)
-#endif
{
#ifdef MALLOC_VALIDATE
return (isalloc_validate(ptr));
#else
assert(ptr != NULL);
return (isalloc(ptr));
#endif
}
+void
+jemalloc_stats(jemalloc_stats_t *stats)
+{
+ size_t i;
+
+ assert(stats != NULL);
+
+ /*
+ * Gather runtime settings.
+ */
+ stats->opt_abort = opt_abort;
+ stats->opt_dss =
+#ifdef MALLOC_DSS
+ opt_dss ? true :
+#endif
+ false;
+ stats->opt_junk =
+#ifdef MALLOC_FILL
+ opt_junk ? true :
+#endif
+ false;
+ stats->opt_mmap =
+#ifdef MALLOC_DSS
+ opt_mmap == false ? false :
+#endif
+ true;
+ stats->opt_utrace =
+#ifdef MALLOC_UTRACE
+ opt_utrace ? true :
+#endif
+ false;
+ stats->opt_sysv =
+#ifdef MALLOC_SYSV
+ opt_sysv ? true :
+#endif
+ false;
+ stats->opt_xmalloc =
+#ifdef MALLOC_XMALLOC
+ opt_xmalloc ? true :
+#endif
+ false;
+ stats->opt_zero =
+#ifdef MALLOC_FILL
+ opt_zero ? true :
+#endif
+ false;
+ stats->narenas = narenas;
+ stats->balance_threshold =
+#ifdef MALLOC_BALANCE
+ opt_balance_threshold
+#else
+ SIZE_T_MAX
+#endif
+ ;
+ stats->quantum = quantum;
+ stats->small_max = small_max;
+ stats->large_max = arena_maxclass;
+ stats->chunksize = chunksize;
+ stats->dirty_max = opt_dirty_max;
+
+ /*
+ * Gather current memory usage statistics.
+ */
+ stats->mapped = 0;
+ stats->committed = 0;
+ stats->allocated = 0;
+ stats->dirty = 0;
+
+ /* Get huge mapped/allocated. */
+ malloc_mutex_lock(&huge_mtx);
+ stats->mapped += stats_chunks.curchunks * chunksize;
+#ifdef MALLOC_DECOMMIT
+ stats->committed += huge_allocated;
+#endif
+ stats->allocated += huge_allocated;
+ malloc_mutex_unlock(&huge_mtx);
+
+ /* Get base mapped. */
+ malloc_mutex_lock(&base_mtx);
+ stats->mapped += base_mapped;
+#ifdef MALLOC_DECOMMIT
+ stats->committed += base_mapped;
+#endif
+ malloc_mutex_unlock(&base_mtx);
+
+ /* Iterate over arenas and their chunks. */
+ for (i = 0; i < narenas; i++) {
+ arena_t *arena = arenas[i];
+ if (arena != NULL) {
+ arena_chunk_t *chunk;
+
+ malloc_spin_lock(&arena->lock);
+ stats->allocated += arena->stats.allocated_small;
+ stats->allocated += arena->stats.allocated_large;
+#ifdef MALLOC_DECOMMIT
+ RB_FOREACH(chunk, arena_chunk_tree_s, &arena->chunks) {
+ size_t j;
+
+ for (j = 0; j < chunk_npages; j++) {
+ if ((chunk->map[j] &
+ CHUNK_MAP_DECOMMITTED) == 0)
+ stats->committed += pagesize;
+ }
+ }
+#endif
+ stats->dirty += (arena->ndirty << pagesize_2pow);
+ malloc_spin_unlock(&arena->lock);
+ }
+ }
+
+#ifndef MALLOC_DECOMMIT
+ stats->committed = stats->mapped;
+#endif
+}
+
#ifdef MOZ_MEMORY_WINDOWS
void*
_recalloc(void *ptr, size_t count, size_t size)
{
size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
size_t newsize = count * size;
/*
@@ -6182,48 +6271,48 @@ zone_size(malloc_zone_t *zone, void *ptr
*/
return (isalloc_validate(ptr));
}
static void *
zone_malloc(malloc_zone_t *zone, size_t size)
{
- return (moz_malloc(size));
+ return (malloc(size));
}
static void *
zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
{
- return (moz_calloc(num, size));
+ return (calloc(num, size));
}
static void *
zone_valloc(malloc_zone_t *zone, size_t size)
{
void *ret = NULL; /* Assignment avoids useless compiler warning. */
- moz_posix_memalign(&ret, pagesize, size);
+ posix_memalign(&ret, pagesize, size);
return (ret);
}
static void
zone_free(malloc_zone_t *zone, void *ptr)
{
- moz_free(ptr);
+ free(ptr);
}
static void *
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
{
- return (moz_realloc(ptr, size));
+ return (realloc(ptr, size));
}
static void *
zone_destroy(malloc_zone_t *zone)
{
/* This function should never be called. */
assert(false);
@@ -6236,20 +6325,20 @@ zone_good_size(malloc_zone_t *zone, size
size_t ret;
void *p;
/*
* Actually create an object of the appropriate size, then find out
* how large it could have been without moving up to the next size
* class.
*/
- p = moz_malloc(size);
+ p = malloc(size);
if (p != NULL) {
ret = isalloc(p);
- moz_free(p);
+ free(p);
} else
ret = size;
return (ret);
}
static void
zone_force_lock(malloc_zone_t *zone)
new file mode 100644
--- /dev/null
+++ b/memory/jemalloc/jemalloc.h
@@ -0,0 +1,57 @@
+#ifndef MOZ_MEMORY_WINDOWS
+# include <stdbool.h>
+#else
+# include <windows.h>
+# ifndef bool
+# define bool BOOL
+# endif
+#endif
+
+extern const char *_malloc_options;
+
+/*
+ * jemalloc_stats() is not a stable interface. When using jemalloc_stats_t, be
+ * sure that the compiled results of jemalloc.c are in sync with this header
+ * file.
+ */
+typedef struct {
+ /*
+ * Run-time configuration settings.
+ */
+ bool opt_abort; /* abort(3) on error? */
+ bool opt_dss; /* Use sbrk(2) to map memory? */
+ bool opt_junk; /* Fill allocated/free memory with 0xa5/0x5a? */
+ bool opt_mmap; /* Use mmap(2) to map memory? */
+ bool opt_utrace; /* Trace all allocation events? */
+ bool opt_sysv; /* SysV semantics? */
+ bool opt_xmalloc; /* abort(3) on OOM? */
+ bool opt_zero; /* Fill allocated memory with 0x0? */
+ size_t narenas; /* Number of arenas. */
+ size_t balance_threshold; /* Arena contention rebalance threshold. */
+ size_t quantum; /* Allocation quantum. */
+ size_t small_max; /* Max quantum-spaced allocation size. */
+ size_t large_max; /* Max sub-chunksize allocation size. */
+ size_t chunksize; /* Size of each virtual memory mapping. */
+ size_t dirty_max; /* Max dirty pages per arena. */
+
+ /*
+ * Current memory usage statistics.
+ */
+ size_t mapped; /* Bytes mapped (not necessarily committed). */
+ size_t committed; /* Bytes committed (readable/writable). */
+ size_t allocated; /* Bytes allocted (in use by application). */
+ size_t dirty; /* Bytes dirty (committed unused pages). */
+} jemalloc_stats_t;
+
+#ifndef MOZ_MEMORY_DARWIN
+void *malloc(size_t size);
+void *valloc(size_t size);
+void *calloc(size_t num, size_t size);
+void *realloc(void *ptr, size_t size);
+void free(void *ptr);
+#endif
+
+int posix_memalign(void **memptr, size_t alignment, size_t size);
+void *memalign(size_t alignment, size_t size);
+size_t malloc_usable_size(const void *ptr);
+void jemalloc_stats(jemalloc_stats_t *stats);