Bug 678977 - Teach sqlite to use jemalloc directly when applicable. code=nnethercote,khuey. r=khuey,sdwilsh,jlebar.

build/unix/gnu-ld-scripts/jemalloc-standalone-linkage-version-script

@@ -4,6 +4,7 @@
 		_malloc_prefork;
 		jemalloc_stats;
 		malloc_usable_size;
+		je_malloc_usable_size_in_advance;
 		posix_memalign;
 		free;
 		realloc;

memory/jemalloc/jemalloc.c

@@ -6495,6 +6495,47 @@ free(void *ptr)
 /*
  * Begin non-standard functions.
  */
+
+/* This was added by Mozilla for use by SQLite. */
+size_t
+je_malloc_usable_size_in_advance(size_t size)
+{
+	/*
+	 * This duplicates the logic in imalloc(), arena_malloc() and
+	 * arena_malloc_small().
+	 */
+	if (size < small_min) {
+		/* Small (tiny). */
+		size = pow2_ceil(size);
+		/*
+		 * We omit the #ifdefs from arena_malloc_small() --
+		 * it can be inaccurate with its size in some cases, but this
+		 * function must be accurate.
+		 */
+		if (size < (1U << TINY_MIN_2POW))
+			size = (1U << TINY_MIN_2POW);
+	} else if (size <= small_max) {
+		/* Small (quantum-spaced). */
+		size = QUANTUM_CEILING(size);
+	} else if (size <= bin_maxclass) {
+		/* Small (sub-page). */
+		size = pow2_ceil(size);
+	} else if (size <= arena_maxclass) {
+		/* Large. */
+		size = PAGE_CEILING(size);
+	} else {
+		/*
+		 * Huge.  We use PAGE_CEILING to get psize, instead of using
+		 * CHUNK_CEILING to get csize.  This ensures that this
+		 * malloc_usable_size(malloc(n)) always matches
+		 * je_malloc_usable_size_in_advance(n).
+		 */
+		size = PAGE_CEILING(size);
+	}
+	return size;
+}
+
+
 #ifdef MOZ_MEMORY_ANDROID
 size_t
 malloc_usable_size(void *ptr)

memory/jemalloc/jemalloc.h

@@ -80,6 +80,9 @@ size_t	malloc_usable_size(const void *ptr);
 
 void	jemalloc_stats(jemalloc_stats_t *stats);
 
+/* Computes the usable size in advance. */
+size_t	je_malloc_usable_size_in_advance(size_t size);
+
 /*
  * On some operating systems (Mac), we use madvise(MADV_FREE) to hand pages
  * back to the operating system.  On Mac, the operating system doesn't take

memory/mozutils/mozutils.def.in

@@ -51,6 +51,7 @@ EXPORTS
   wcsdup=je_wcsdup
   _wcsdup=je_wcsdup
   malloc_usable_size=je_malloc_usable_size
+  je_malloc_usable_size_in_advance
   jemalloc_stats
   ; A hack to work around the CRT (see giant comment in Makefile.in)
   frex=je_dumb_free_thunk

storage/src/Makefile.in

@@ -50,6 +50,10 @@ FORCE_STATIC_LIB = 1
 GRE_MODULE       = 1
 LIBXUL_LIBRARY = 1
 
+# TODO: we do this in crashreporter and xpcom/base too, should be centralized
+ifeq ($(OS_ARCH),Linux)
+DEFINES += -DXP_LINUX
+endif
 
 EXPORTS_NAMESPACES = mozilla/storage
 

storage/src/mozStorageService.cpp

@@ -314,7 +314,91 @@ Service::shutdown()
 }
 
 sqlite3_vfs* ConstructTelemetryVFS();
- 
+
+#ifdef MOZ_MEMORY
+
+#  if defined(XP_WIN) || defined(SOLARIS) || defined(ANDROID) || defined(XP_MACOSX)
+#    include "jemalloc.h"
+#  elif defined(XP_LINUX)
+// jemalloc is directly linked into firefox-bin; libxul doesn't link
+// with it.  So if we tried to use je_malloc_usable_size_in_advance directly
+// here, it wouldn't be defined.  Instead, we don't include the jemalloc header
+// and weakly link against je_malloc_usable_size_in_advance.
+extern "C" {
+extern size_t je_malloc_usable_size_in_advance(size_t size)
+  NS_VISIBILITY_DEFAULT __attribute__((weak));
+}
+#  endif  // XP_LINUX
+
+namespace {
+
+// By default, SQLite tracks the size of all its heap blocks by adding an extra
+// 8 bytes at the start of the block to hold the size.  Unfortunately, this
+// causes a lot of 2^N-sized allocations to be rounded up by jemalloc
+// allocator, wasting memory.  For example, a request for 1024 bytes has 8
+// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up
+// to 2048 bytes, wasting 1012 bytes.  (See bug 676189 for more details.)
+//
+// So we register jemalloc as the malloc implementation, which avoids this
+// 8-byte overhead, and thus a lot of waste.  This requires us to provide a
+// function, sqliteMemRoundup(), which computes the actual size that will be
+// allocated for a given request.  SQLite uses this function before all
+// allocations, and may be able to use any excess bytes caused by the rounding.
+//
+// Note: the wrappers for moz_malloc, moz_realloc and moz_malloc_usable_size
+// are necessary because the sqlite_mem_methods type signatures differ slightly
+// from the standard ones -- they use int instead of size_t.  But we don't need
+// a wrapper for moz_free.
+
+static void* sqliteMemMalloc(int n)
+{
+  return ::moz_malloc(n);
+}
+
+static void* sqliteMemRealloc(void* p, int n)
+{
+  return ::moz_realloc(p, n);
+}
+
+static int sqliteMemSize(void* p)
+{
+  return ::moz_malloc_usable_size(p);
+}
+
+static int sqliteMemRoundup(int n)
+{
+  n = je_malloc_usable_size_in_advance(n);
+
+  // jemalloc can return blocks of size 2 and 4, but SQLite requires that all
+  // allocations be 8-aligned.  So we round up sub-8 requests to 8.  This
+  // wastes a small amount of memory but is obviously safe.
+  return n <= 8 ? 8 : n;
+}
+
+static int sqliteMemInit(void* p)
+{
+  return 0;
+}
+
+static void sqliteMemShutdown(void* p)
+{
+}
+
+const sqlite3_mem_methods memMethods = {
+  &sqliteMemMalloc,
+  &moz_free,
+  &sqliteMemRealloc,
+  &sqliteMemSize,
+  &sqliteMemRoundup,
+  &sqliteMemInit,
+  &sqliteMemShutdown,
+  NULL
+}; 
+
+} // anonymous namespace
+
+#endif  // MOZ_MEMORY
+
 nsresult
 Service::initialize()
 {
@@ -322,6 +406,12 @@ Service::initialize()
 
   int rc;
 
+#ifdef MOZ_MEMORY
+  rc = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods);
+  if (rc != SQLITE_OK)
+    return convertResultCode(rc);
+#endif
+
   // Explicitly initialize sqlite3.  Although this is implicitly called by
   // various sqlite3 functions (and the sqlite3_open calls in our case),
   // the documentation suggests calling this directly.  So we do.

xpcom/base/Makefile.in

@@ -49,7 +49,7 @@ GRE_MODULE	= 1
 MOZILLA_INTERNAL_API =1
 LIBXUL_LIBRARY = 1
 
-# TODO: we do this in crashreporter too, should be centralized
+# TODO: we do this in crashreporter and storage/src too, should be centralized
 ifeq ($(OS_ARCH),Linux)
 DEFINES += -DXP_LINUX
 endif

xpcom/tests/Makefile.in

@@ -101,6 +101,10 @@ CPP_UNIT_TESTS = \
                  TestTArray.cpp \
                  $(NULL)
 
+ifdef MOZ_MEMORY
+CPP_UNIT_TESTS += TestJemalloc.cpp
+endif
+
 # XXX Make this tests work in libxul builds.
 #CPP_UNIT_TESTS += \
 #                  TestArray.cpp \

xpcom/tests/TestJemalloc.cpp

@@ -0,0 +1,111 @@
+/* -*-  Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is c++ array template tests.
+ *
+ * The Initial Developer of the Original Code is
+ * The Mozilla Foundation <http://www.mozilla.org/>.
+ * Portions created by the Initial Developer are Copyright (C) 2011
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *  Nicholas Nethercote <nnethercote@mozilla.com>
+ *  
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/*
+ * Ideally, this test would be in memory/test/.  But I couldn't get it to build
+ * there (couldn't find TestHarness.h).  I think memory/ is processed too early
+ * in the build.  So it's here.
+ */
+
+#include "TestHarness.h"
+#include "jemalloc.h"
+
+static inline bool
+TestOne(size_t size)
+{
+    size_t req = size;
+    size_t adv = je_malloc_usable_size_in_advance(req);
+    char* p = (char*)malloc(req);
+    size_t usable = moz_malloc_usable_size(p);
+    if (adv != usable) {
+      fail("je_malloc_usable_size_in_advance(%d) --> %d; "
+           "malloc_usable_size(%d) --> %d",
+           req, adv, req, usable);
+      return false;
+    }
+    free(p);
+    return true;
+}
+
+static inline bool
+TestThree(size_t size)
+{
+    return TestOne(size - 1) && TestOne(size) && TestOne(size + 1);
+}
+
+static nsresult
+TestJemallocUsableSizeInAdvance()
+{
+  #define K   * 1024
+  #define M   * 1024 * 1024
+
+  /*
+   * Test every size up to a certain point, then (N-1, N, N+1) triplets for a
+   * various sizes beyond that.
+   */
+
+  for (size_t n = 0; n < 16 K; n++)
+    if (!TestOne(n))
+      return NS_ERROR_UNEXPECTED;
+
+  for (size_t n = 16 K; n < 1 M; n += 4 K)
+    if (!TestThree(n))
+      return NS_ERROR_UNEXPECTED;
+
+  for (size_t n = 1 M; n < 8 M; n += 128 K)
+    if (!TestThree(n))
+      return NS_ERROR_UNEXPECTED;
+
+  passed("je_malloc_usable_size_in_advance");
+
+  return NS_OK;
+}
+
+int main(int argc, char** argv)
+{
+  int rv = 0;
+  ScopedXPCOM xpcom("jemalloc");
+  if (xpcom.failed())
+      return 1;
+
+  if (NS_FAILED(TestJemallocUsableSizeInAdvance()))
+    rv = 1;
+
+  return rv;
+}
+