Bug 1168719 - Generic replace-malloc library. r=njn

memory/build/malloc_decls.h

@@ -35,6 +35,10 @@ typedef MALLOC_USABLE_SIZE_CONST_PTR void * usable_ptr_t;
 #endif
 
 #ifdef MALLOC_DECL
+#  ifndef MALLOC_DECL_VOID
+#    define MALLOC_DECL_VOID(func, ...) MALLOC_DECL(func, void, __VA_ARGS__)
+#  endif
+
 #  if MALLOC_FUNCS & MALLOC_FUNCS_INIT
 MALLOC_DECL(init, void, const malloc_table_t *)
 #  endif
@@ -47,17 +51,19 @@ MALLOC_DECL(posix_memalign, int, void **, size_t, size_t)
 MALLOC_DECL(aligned_alloc, void *, size_t, size_t)
 MALLOC_DECL(calloc, void *, size_t, size_t)
 MALLOC_DECL(realloc, void *, void *, size_t)
-MALLOC_DECL(free, void, void *)
+MALLOC_DECL_VOID(free, void *)
 MALLOC_DECL(memalign, void *, size_t, size_t)
 MALLOC_DECL(valloc, void *, size_t)
 MALLOC_DECL(malloc_usable_size, size_t, usable_ptr_t)
 MALLOC_DECL(malloc_good_size, size_t, size_t)
 #  endif
 #  if MALLOC_FUNCS & MALLOC_FUNCS_JEMALLOC
-MALLOC_DECL(jemalloc_stats, void, jemalloc_stats_t *)
-MALLOC_DECL(jemalloc_purge_freed_pages, void, void)
-MALLOC_DECL(jemalloc_free_dirty_pages, void, void)
+MALLOC_DECL_VOID(jemalloc_stats, jemalloc_stats_t *)
+MALLOC_DECL_VOID(jemalloc_purge_freed_pages, void)
+MALLOC_DECL_VOID(jemalloc_free_dirty_pages, void)
 #  endif
+
+#  undef MALLOC_DECL_VOID
 #endif /* MALLOC_DECL */
 
 #undef MALLOC_DECL

memory/build/replace_malloc.h

@@ -74,19 +74,6 @@
 
 MOZ_BEGIN_EXTERN_C
 
-#define MALLOC_DECL(name, return_type, ...) \
-  typedef return_type(name ## _impl_t)(__VA_ARGS__);
-
-#include "malloc_decls.h"
-
-#define MALLOC_DECL(name, return_type, ...) \
-  name ## _impl_t * name;
-
-typedef struct {
-#include "malloc_decls.h"
-} malloc_table_t;
-
-
 /* MOZ_NO_REPLACE_FUNC_DECL and MOZ_REPLACE_WEAK are only defined in
  * replace_malloc.c. Normally including this header will add function
  * definitions. */

memory/build/replace_malloc_bridge.h

@@ -48,15 +48,59 @@
 
 struct ReplaceMallocBridge;
 
-#ifdef __cplusplus
-
 #include "mozilla/Types.h"
 
+MOZ_BEGIN_EXTERN_C
+
 #ifndef REPLACE_MALLOC_IMPL
 /* Returns the replace-malloc bridge if there is one to be returned. */
-extern "C" MFBT_API ReplaceMallocBridge* get_bridge();
+MFBT_API ReplaceMallocBridge* get_bridge();
 #endif
 
+/* Table of malloc functions.
+ *   e.g. void* (*malloc)(size_t), etc.
+ */
+#define MALLOC_DECL(name, return_type, ...) \
+  typedef return_type(name ## _impl_t)(__VA_ARGS__);
+
+#include "malloc_decls.h"
+
+#define MALLOC_DECL(name, return_type, ...) \
+  name ## _impl_t * name;
+
+typedef struct {
+#include "malloc_decls.h"
+} malloc_table_t;
+
+
+/* Table of malloc hook functions.
+ * Those functions are called with the arguments and results of malloc
+ * functions after they are called.
+ *   e.g. void* (*malloc_hook)(void*, size_t), etc.
+ * They can either return the result they're given, or alter it before
+ * returning it.
+ * The hooks corresponding to functions, like free(void*), that return no
+ * value, don't take an extra argument.
+ * The table must at least contain a pointer for malloc_hook and free_hook
+ * functions. They will be used as fallback if no pointer is given for
+ * other allocation functions, like calloc_hook.
+ */
+#define MALLOC_DECL(name, return_type, ...) \
+  return_type (*name ## _hook)(return_type, __VA_ARGS__);
+#define MALLOC_DECL_VOID(name, ...) \
+  void (*name ## _hook)(__VA_ARGS__);
+
+typedef struct {
+#include "malloc_decls.h"
+  /* Like free_hook, but called before realloc_hook. free_hook is called
+   * instead of not given. */
+  void (*realloc_hook_before)(void* aPtr);
+} malloc_hook_table_t;
+
+MOZ_END_EXTERN_C
+
+#ifdef __cplusplus
+
 namespace mozilla {
 namespace dmd {
 struct DMDFuncs;
@@ -75,7 +119,7 @@ struct DebugFdRegistry
 
 struct ReplaceMallocBridge
 {
-  ReplaceMallocBridge() : mVersion(2) {}
+  ReplaceMallocBridge() : mVersion(3) {}
 
   /* This method was added in version 1 of the bridge. */
   virtual mozilla::dmd::DMDFuncs* GetDMDFuncs() { return nullptr; }
@@ -86,6 +130,23 @@ struct ReplaceMallocBridge
    * This method was added in version 2 of the bridge. */
   virtual void InitDebugFd(mozilla::DebugFdRegistry&) {}
 
+  /* Register a list of malloc functions and hook functions to the
+   * replace-malloc library so that it can choose to dispatch to them
+   * when needed. The details of what is dispatched when is left to the
+   * replace-malloc library.
+   * Passing a nullptr for either table will unregister a previously
+   * registered table under the same name.
+   * Returns nullptr if registration failed.
+   * If registration succeeded, a table of "pure" malloc functions is
+   * returned. Those "pure" malloc functions won't call hooks.
+   * /!\ Do not rely on registration/unregistration to be instantaneous.
+   * Functions from a previously registered table may still be called for
+   * a brief time after RegisterHook returns.
+   * This method was added in version 3 of the bridge. */
+  virtual const malloc_table_t*
+  RegisterHook(const char* aName, const malloc_table_t* aTable,
+               const malloc_hook_table_t* aHookTable) { return nullptr; }
+
 #ifndef REPLACE_MALLOC_IMPL
   /* Returns the replace-malloc bridge if its version is at least the
    * requested one. */
@@ -124,6 +185,15 @@ struct ReplaceMalloc
       singleton->InitDebugFd(aRegistry);
     }
   }
+
+  static const malloc_table_t*
+  RegisterHook(const char* aName, const malloc_table_t* aTable,
+               const malloc_hook_table_t* aHookTable)
+  {
+    auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 3);
+    return singleton ? singleton->RegisterHook(aName, aTable, aHookTable)
+                     : nullptr;
+  }
 };
 #endif
 

memory/replace/moz.build

@@ -4,7 +4,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += ['logalloc']
+DIRS += [
+    'logalloc',
+    'replace',
+]
 
 # Build jemalloc3 as a replace-malloc lib when building with mozjemalloc
 if not CONFIG['MOZ_JEMALLOC3']:

memory/replace/replace/ReplaceMalloc.cpp

@@ -0,0 +1,255 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "replace_malloc.h"
+#include <errno.h>
+#include "mozilla/CheckedInt.h"
+#include "mozilla/Atomics.h"
+
+/* Replace-malloc library allowing different kinds of dispatch.
+ * The long term goal is to allow multiple replace-malloc libraries
+ * to be loaded and coexist properly.
+ * This is however a limited version to fulfil more immediate needs.
+ */
+static const malloc_table_t* gFuncs = nullptr;
+/* This should normally be a mozilla::Atomic<const malloc_hook_table_t*>
+ * but MSVC 2013's atomic type doesn't like it. */
+static mozilla::Atomic<malloc_hook_table_t*> gHookTable(nullptr);
+
+class GenericReplaceMallocBridge : public ReplaceMallocBridge
+{
+  virtual const malloc_table_t*
+  RegisterHook(const char* aName, const malloc_table_t* aTable,
+               const malloc_hook_table_t* aHookTable) override
+  {
+    // Can't register a hook before replace_init is called.
+    if (!gFuncs) {
+      return nullptr;
+    }
+
+    // Expect a name to be given.
+    if (!aName) {
+      return nullptr;
+    }
+
+    // Giving a malloc_table_t is not supported yet.
+    if (aTable) {
+      return nullptr;
+    }
+
+    if (aHookTable) {
+      // Expect at least a malloc and a free hook.
+      if (!aHookTable->malloc_hook || !aHookTable->free_hook) {
+        return nullptr;
+      }
+      gHookTable = const_cast<malloc_hook_table_t*>(aHookTable);
+      return gFuncs;
+    }
+    gHookTable = nullptr;
+
+    return nullptr;
+  }
+};
+
+void
+replace_init(const malloc_table_t* aTable)
+{
+  gFuncs = aTable;
+}
+
+ReplaceMallocBridge*
+replace_get_bridge()
+{
+  static GenericReplaceMallocBridge bridge;
+  return &bridge;
+}
+
+void*
+replace_malloc(size_t aSize)
+{
+  void* ptr = gFuncs->malloc(aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    return hook_table->malloc_hook(ptr, aSize);
+  }
+  return ptr;
+}
+
+int
+replace_posix_memalign(void** aPtr, size_t aAlignment, size_t aSize)
+{
+  int ret = gFuncs->posix_memalign(aPtr, aAlignment, aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->posix_memalign_hook) {
+      return hook_table->posix_memalign_hook(ret, aPtr, aAlignment, aSize);
+    }
+    void* ptr = hook_table->malloc_hook(*aPtr, aSize);
+    if (!ptr && *aPtr) {
+      *aPtr = ptr;
+      ret = ENOMEM;
+    }
+  }
+  return ret;
+}
+
+void*
+replace_aligned_alloc(size_t aAlignment, size_t aSize)
+{
+  void* ptr = gFuncs->aligned_alloc(aAlignment, aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->aligned_alloc_hook) {
+      return hook_table->aligned_alloc_hook(ptr, aAlignment, aSize);
+    }
+    return hook_table->malloc_hook(ptr, aSize);
+  }
+  return ptr;
+}
+
+void*
+replace_calloc(size_t aNum, size_t aSize)
+{
+  void* ptr = gFuncs->calloc(aNum, aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->calloc_hook) {
+      return hook_table->calloc_hook(ptr, aNum, aSize);
+    }
+    mozilla::CheckedInt<size_t> size = mozilla::CheckedInt<size_t>(aNum) * aSize;
+    if (size.isValid()) {
+      return hook_table->malloc_hook(ptr, size.value());
+    }
+    /* If the multiplication above overflows, calloc will have failed, so ptr
+     * is null. But the hook might still be interested in knowing about the
+     * allocation attempt. The choice made is to indicate the overflow with
+     * the biggest value of a size_t, which is not that bad an indicator:
+     * there are only 5 prime factors to 2^32 - 1 and 7 prime factors to
+     * 2^64 - 1 and none of them is going to come directly out of sizeof().
+     * IOW, the likelyhood of aNum * aSize being exactly SIZE_MAX is low
+     * enough, and SIZE_MAX still conveys that the attempted allocation was
+     * too big anyways. */
+    return hook_table->malloc_hook(ptr, SIZE_MAX);
+  }
+  return ptr;
+}
+
+void*
+replace_realloc(void* aPtr, size_t aSize)
+{
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->realloc_hook_before) {
+      hook_table->realloc_hook_before(aPtr);
+    } else {
+      hook_table->free_hook(aPtr);
+    }
+  }
+  void* new_ptr = gFuncs->realloc(aPtr, aSize);
+  /* The hook table might have changed since before realloc was called,
+   * either because of unregistration or registration of a new table.
+   * We however go with consistency and use the same hook table as the
+   * one that was used before the call to realloc. */
+  if (hook_table) {
+    if (hook_table->realloc_hook) {
+      /* aPtr is likely invalid when reaching here, it is only given for
+       * tracking purposes, and should not be dereferenced. */
+      return hook_table->realloc_hook(new_ptr, aPtr, aSize);
+    }
+    return hook_table->malloc_hook(new_ptr, aSize);
+  }
+  return new_ptr;
+}
+
+void
+replace_free(void* aPtr)
+{
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    hook_table->free_hook(aPtr);
+  }
+  gFuncs->free(aPtr);
+}
+
+void*
+replace_memalign(size_t aAlignment, size_t aSize)
+{
+  void* ptr = gFuncs->memalign(aAlignment, aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->memalign_hook) {
+      return hook_table->memalign_hook(ptr, aAlignment, aSize);
+    }
+    return hook_table->malloc_hook(ptr, aSize);
+  }
+  return ptr;
+}
+
+void*
+replace_valloc(size_t aSize)
+{
+  void* ptr = gFuncs->valloc(aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table) {
+    if (hook_table->valloc_hook) {
+      return hook_table->valloc_hook(ptr, aSize);
+    }
+    return hook_table->malloc_hook(ptr, aSize);
+  }
+  return ptr;
+}
+
+size_t
+replace_malloc_usable_size(usable_ptr_t aPtr)
+{
+  size_t ret = gFuncs->malloc_usable_size(aPtr);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table && hook_table->malloc_usable_size_hook) {
+    return hook_table->malloc_usable_size_hook(ret, aPtr);
+  }
+  return ret;
+}
+
+size_t
+replace_malloc_good_size(size_t aSize)
+{
+  size_t ret = gFuncs->malloc_good_size(aSize);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table && hook_table->malloc_good_size_hook) {
+    return hook_table->malloc_good_size_hook(ret, aSize);
+  }
+  return ret;
+}
+
+void
+replace_jemalloc_stats(jemalloc_stats_t* aStats)
+{
+  gFuncs->jemalloc_stats(aStats);
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table && hook_table->jemalloc_stats_hook) {
+    hook_table->jemalloc_stats_hook(aStats);
+  }
+}
+
+void
+replace_jemalloc_purge_freed_pages(void)
+{
+  gFuncs->jemalloc_purge_freed_pages();
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table && hook_table->jemalloc_purge_freed_pages_hook) {
+    hook_table->jemalloc_purge_freed_pages_hook();
+  }
+}
+
+void
+replace_jemalloc_free_dirty_pages(void)
+{
+  gFuncs->jemalloc_free_dirty_pages();
+  const malloc_hook_table_t* hook_table = gHookTable;
+  if (hook_table && hook_table->jemalloc_free_dirty_pages_hook) {
+    hook_table->jemalloc_free_dirty_pages_hook();
+  }
+}

memory/replace/replace/moz.build

@@ -0,0 +1,13 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SharedLibrary('replace_malloc')
+
+SOURCES += [
+    'ReplaceMalloc.cpp',
+]
+
+DISABLE_STL_WRAPPING = True