From 6313e2098306d7f0a78874848e4af3fb2f524a2f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 3 Nov 2011 20:53:41 -0700 Subject: [PATCH] Bug 678977 - Teach sqlite to use jemalloc directly when applicable. code=nnethercote,khuey. r=khuey,sdwilsh,jlebar. --- ...jemalloc-standalone-linkage-version-script | 1 + memory/jemalloc/jemalloc.c | 41 +++++++ memory/jemalloc/jemalloc.h | 3 + memory/mozutils/mozutils.def.in | 1 + storage/src/Makefile.in | 4 + storage/src/mozStorageService.cpp | 92 ++++++++++++++- xpcom/base/Makefile.in | 2 +- xpcom/tests/Makefile.in | 4 + xpcom/tests/TestJemalloc.cpp | 111 ++++++++++++++++++ 9 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 xpcom/tests/TestJemalloc.cpp diff --git a/build/unix/gnu-ld-scripts/jemalloc-standalone-linkage-version-script b/build/unix/gnu-ld-scripts/jemalloc-standalone-linkage-version-script index 59b3a4c1e66..3e3d827ff6e 100644 --- a/build/unix/gnu-ld-scripts/jemalloc-standalone-linkage-version-script +++ b/build/unix/gnu-ld-scripts/jemalloc-standalone-linkage-version-script @@ -4,6 +4,7 @@ _malloc_prefork; jemalloc_stats; malloc_usable_size; + je_malloc_usable_size_in_advance; posix_memalign; free; realloc; diff --git a/memory/jemalloc/jemalloc.c b/memory/jemalloc/jemalloc.c index ab7aa473cb2..85c15803cf6 100644 --- a/memory/jemalloc/jemalloc.c +++ b/memory/jemalloc/jemalloc.c @@ -6495,6 +6495,47 @@ free(void *ptr) /* * Begin non-standard functions. */ + +/* This was added by Mozilla for use by SQLite. */ +size_t +je_malloc_usable_size_in_advance(size_t size) +{ + /* + * This duplicates the logic in imalloc(), arena_malloc() and + * arena_malloc_small(). + */ + if (size < small_min) { + /* Small (tiny). */ + size = pow2_ceil(size); + /* + * We omit the #ifdefs from arena_malloc_small() -- + * it can be inaccurate with its size in some cases, but this + * function must be accurate. + */ + if (size < (1U << TINY_MIN_2POW)) + size = (1U << TINY_MIN_2POW); + } else if (size <= small_max) { + /* Small (quantum-spaced). */ + size = QUANTUM_CEILING(size); + } else if (size <= bin_maxclass) { + /* Small (sub-page). */ + size = pow2_ceil(size); + } else if (size <= arena_maxclass) { + /* Large. */ + size = PAGE_CEILING(size); + } else { + /* + * Huge. We use PAGE_CEILING to get psize, instead of using + * CHUNK_CEILING to get csize. This ensures that this + * malloc_usable_size(malloc(n)) always matches + * je_malloc_usable_size_in_advance(n). + */ + size = PAGE_CEILING(size); + } + return size; +} + + #ifdef MOZ_MEMORY_ANDROID size_t malloc_usable_size(void *ptr) diff --git a/memory/jemalloc/jemalloc.h b/memory/jemalloc/jemalloc.h index 19817f59460..e46e554cba3 100644 --- a/memory/jemalloc/jemalloc.h +++ b/memory/jemalloc/jemalloc.h @@ -80,6 +80,9 @@ size_t malloc_usable_size(const void *ptr); void jemalloc_stats(jemalloc_stats_t *stats); +/* Computes the usable size in advance. */ +size_t je_malloc_usable_size_in_advance(size_t size); + /* * On some operating systems (Mac), we use madvise(MADV_FREE) to hand pages * back to the operating system. On Mac, the operating system doesn't take diff --git a/memory/mozutils/mozutils.def.in b/memory/mozutils/mozutils.def.in index 92c71f7e253..3bbc3bf09ee 100644 --- a/memory/mozutils/mozutils.def.in +++ b/memory/mozutils/mozutils.def.in @@ -51,6 +51,7 @@ EXPORTS wcsdup=je_wcsdup _wcsdup=je_wcsdup malloc_usable_size=je_malloc_usable_size + je_malloc_usable_size_in_advance jemalloc_stats ; A hack to work around the CRT (see giant comment in Makefile.in) frex=je_dumb_free_thunk diff --git a/storage/src/Makefile.in b/storage/src/Makefile.in index b5aa6a3dd4a..259361f79b0 100644 --- a/storage/src/Makefile.in +++ b/storage/src/Makefile.in @@ -50,6 +50,10 @@ FORCE_STATIC_LIB = 1 GRE_MODULE = 1 LIBXUL_LIBRARY = 1 +# TODO: we do this in crashreporter and xpcom/base too, should be centralized +ifeq ($(OS_ARCH),Linux) +DEFINES += -DXP_LINUX +endif EXPORTS_NAMESPACES = mozilla/storage diff --git a/storage/src/mozStorageService.cpp b/storage/src/mozStorageService.cpp index 75c0e3effe7..7c0a7a0eb5e 100644 --- a/storage/src/mozStorageService.cpp +++ b/storage/src/mozStorageService.cpp @@ -314,7 +314,91 @@ Service::shutdown() } sqlite3_vfs* ConstructTelemetryVFS(); - + +#ifdef MOZ_MEMORY + +# if defined(XP_WIN) || defined(SOLARIS) || defined(ANDROID) || defined(XP_MACOSX) +# include "jemalloc.h" +# elif defined(XP_LINUX) +// jemalloc is directly linked into firefox-bin; libxul doesn't link +// with it. So if we tried to use je_malloc_usable_size_in_advance directly +// here, it wouldn't be defined. Instead, we don't include the jemalloc header +// and weakly link against je_malloc_usable_size_in_advance. +extern "C" { +extern size_t je_malloc_usable_size_in_advance(size_t size) + NS_VISIBILITY_DEFAULT __attribute__((weak)); +} +# endif // XP_LINUX + +namespace { + +// By default, SQLite tracks the size of all its heap blocks by adding an extra +// 8 bytes at the start of the block to hold the size. Unfortunately, this +// causes a lot of 2^N-sized allocations to be rounded up by jemalloc +// allocator, wasting memory. For example, a request for 1024 bytes has 8 +// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up +// to 2048 bytes, wasting 1012 bytes. (See bug 676189 for more details.) +// +// So we register jemalloc as the malloc implementation, which avoids this +// 8-byte overhead, and thus a lot of waste. This requires us to provide a +// function, sqliteMemRoundup(), which computes the actual size that will be +// allocated for a given request. SQLite uses this function before all +// allocations, and may be able to use any excess bytes caused by the rounding. +// +// Note: the wrappers for moz_malloc, moz_realloc and moz_malloc_usable_size +// are necessary because the sqlite_mem_methods type signatures differ slightly +// from the standard ones -- they use int instead of size_t. But we don't need +// a wrapper for moz_free. + +static void* sqliteMemMalloc(int n) +{ + return ::moz_malloc(n); +} + +static void* sqliteMemRealloc(void* p, int n) +{ + return ::moz_realloc(p, n); +} + +static int sqliteMemSize(void* p) +{ + return ::moz_malloc_usable_size(p); +} + +static int sqliteMemRoundup(int n) +{ + n = je_malloc_usable_size_in_advance(n); + + // jemalloc can return blocks of size 2 and 4, but SQLite requires that all + // allocations be 8-aligned. So we round up sub-8 requests to 8. This + // wastes a small amount of memory but is obviously safe. + return n <= 8 ? 8 : n; +} + +static int sqliteMemInit(void* p) +{ + return 0; +} + +static void sqliteMemShutdown(void* p) +{ +} + +const sqlite3_mem_methods memMethods = { + &sqliteMemMalloc, + &moz_free, + &sqliteMemRealloc, + &sqliteMemSize, + &sqliteMemRoundup, + &sqliteMemInit, + &sqliteMemShutdown, + NULL +}; + +} // anonymous namespace + +#endif // MOZ_MEMORY + nsresult Service::initialize() { @@ -322,6 +406,12 @@ Service::initialize() int rc; +#ifdef MOZ_MEMORY + rc = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods); + if (rc != SQLITE_OK) + return convertResultCode(rc); +#endif + // Explicitly initialize sqlite3. Although this is implicitly called by // various sqlite3 functions (and the sqlite3_open calls in our case), // the documentation suggests calling this directly. So we do. diff --git a/xpcom/base/Makefile.in b/xpcom/base/Makefile.in index ee2ea8cb627..e9301445e1b 100644 --- a/xpcom/base/Makefile.in +++ b/xpcom/base/Makefile.in @@ -49,7 +49,7 @@ GRE_MODULE = 1 MOZILLA_INTERNAL_API =1 LIBXUL_LIBRARY = 1 -# TODO: we do this in crashreporter too, should be centralized +# TODO: we do this in crashreporter and storage/src too, should be centralized ifeq ($(OS_ARCH),Linux) DEFINES += -DXP_LINUX endif diff --git a/xpcom/tests/Makefile.in b/xpcom/tests/Makefile.in index 3889000ad07..f068a78274c 100644 --- a/xpcom/tests/Makefile.in +++ b/xpcom/tests/Makefile.in @@ -101,6 +101,10 @@ CPP_UNIT_TESTS = \ TestTArray.cpp \ $(NULL) +ifdef MOZ_MEMORY +CPP_UNIT_TESTS += TestJemalloc.cpp +endif + # XXX Make this tests work in libxul builds. #CPP_UNIT_TESTS += \ # TestArray.cpp \ diff --git a/xpcom/tests/TestJemalloc.cpp b/xpcom/tests/TestJemalloc.cpp new file mode 100644 index 00000000000..6ff34715922 --- /dev/null +++ b/xpcom/tests/TestJemalloc.cpp @@ -0,0 +1,111 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is c++ array template tests. + * + * The Initial Developer of the Original Code is + * The Mozilla Foundation . + * Portions created by the Initial Developer are Copyright (C) 2011 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Nicholas Nethercote + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +/* + * Ideally, this test would be in memory/test/. But I couldn't get it to build + * there (couldn't find TestHarness.h). I think memory/ is processed too early + * in the build. So it's here. + */ + +#include "TestHarness.h" +#include "jemalloc.h" + +static inline bool +TestOne(size_t size) +{ + size_t req = size; + size_t adv = je_malloc_usable_size_in_advance(req); + char* p = (char*)malloc(req); + size_t usable = moz_malloc_usable_size(p); + if (adv != usable) { + fail("je_malloc_usable_size_in_advance(%d) --> %d; " + "malloc_usable_size(%d) --> %d", + req, adv, req, usable); + return false; + } + free(p); + return true; +} + +static inline bool +TestThree(size_t size) +{ + return TestOne(size - 1) && TestOne(size) && TestOne(size + 1); +} + +static nsresult +TestJemallocUsableSizeInAdvance() +{ + #define K * 1024 + #define M * 1024 * 1024 + + /* + * Test every size up to a certain point, then (N-1, N, N+1) triplets for a + * various sizes beyond that. + */ + + for (size_t n = 0; n < 16 K; n++) + if (!TestOne(n)) + return NS_ERROR_UNEXPECTED; + + for (size_t n = 16 K; n < 1 M; n += 4 K) + if (!TestThree(n)) + return NS_ERROR_UNEXPECTED; + + for (size_t n = 1 M; n < 8 M; n += 128 K) + if (!TestThree(n)) + return NS_ERROR_UNEXPECTED; + + passed("je_malloc_usable_size_in_advance"); + + return NS_OK; +} + +int main(int argc, char** argv) +{ + int rv = 0; + ScopedXPCOM xpcom("jemalloc"); + if (xpcom.failed()) + return 1; + + if (NS_FAILED(TestJemallocUsableSizeInAdvance())) + rv = 1; + + return rv; +} +