Bug 866846 - Use WAL journal mode for IndexedDB databases, r=janv.

dom/quota/QuotaObject.cpp

@@ -6,11 +6,62 @@
 
 #include "QuotaObject.h"
 
+#include "mozilla/TypeTraits.h"
 #include "QuotaManager.h"
 #include "Utilities.h"
 
+#ifdef DEBUG
+#include "nsComponentManagerUtils.h"
+#include "nsIFile.h"
+#include "nsXPCOMCID.h"
+#endif
+
 USING_QUOTA_NAMESPACE
 
+namespace {
+
+template <typename T, typename U>
+void
+AssertPositiveIntegers(T aOne, U aTwo)
+{
+  static_assert(mozilla::IsIntegral<T>::value, "Not an integer!");
+  static_assert(mozilla::IsIntegral<U>::value, "Not an integer!");
+  MOZ_ASSERT(aOne >= 0);
+  MOZ_ASSERT(aTwo >= 0);
+}
+
+template <typename T, typename U>
+void
+AssertNoOverflow(T aOne, U aTwo)
+{
+  AssertPositiveIntegers(aOne, aTwo);
+  AssertNoOverflow(uint64_t(aOne), uint64_t(aTwo));
+}
+
+template <>
+void
+AssertNoOverflow<uint64_t, uint64_t>(uint64_t aOne, uint64_t aTwo)
+{
+  MOZ_ASSERT(UINT64_MAX - aOne >= aTwo);
+}
+
+template <typename T, typename U>
+void
+AssertNoUnderflow(T aOne, U aTwo)
+{
+  AssertPositiveIntegers(aOne, aTwo);
+  AssertNoUnderflow(uint64_t(aOne), uint64_t(aTwo));
+}
+
+template <>
+void
+AssertNoUnderflow<uint64_t, uint64_t>(uint64_t aOne, uint64_t aTwo)
+{
+  MOZ_ASSERT(aOne >= aTwo);
+}
+
+} // anonymous namespace
+
 void
 QuotaObject::AddRef()
 {
@@ -64,31 +115,65 @@ QuotaObject::Release()
 void
 QuotaObject::UpdateSize(int64_t aSize)
 {
+  MOZ_ASSERT(aSize >= 0);
+
+#ifdef DEBUG
+  {
+    nsCOMPtr<nsIFile> file = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID);
+    MOZ_ASSERT(file);
+
+    MOZ_ASSERT(NS_SUCCEEDED(file->InitWithPath(mPath)));
+
+    bool exists;
+    MOZ_ASSERT(NS_SUCCEEDED(file->Exists(&exists)));
+
+    if (exists) {
+      int64_t fileSize;
+      MOZ_ASSERT(NS_SUCCEEDED(file->GetFileSize(&fileSize)));
+
+      MOZ_ASSERT(aSize == fileSize);
+    } else {
+      MOZ_ASSERT(!aSize);
+    }
+  }
+#endif
+
   QuotaManager* quotaManager = QuotaManager::Get();
   NS_ASSERTION(quotaManager, "Shouldn't be null!");
 
   MutexAutoLock lock(quotaManager->mQuotaMutex);
 
-  if (!mOriginInfo) {
+  if (!mOriginInfo || mSize == aSize) {
     return;
   }
 
+  AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, mSize);
+  quotaManager->mTemporaryStorageUsage -= mSize;
+
   GroupInfo* groupInfo = mOriginInfo->mGroupInfo;
 
-  quotaManager->mTemporaryStorageUsage -= mSize;
+  AssertNoUnderflow(groupInfo->mUsage, mSize);
   groupInfo->mUsage -= mSize;
+
+  AssertNoUnderflow(mOriginInfo->mUsage, mSize);
   mOriginInfo->mUsage -= mSize;
 
   mSize = aSize;
 
+  AssertNoOverflow(mOriginInfo->mUsage, mSize);
   mOriginInfo->mUsage += mSize;
+
+  AssertNoOverflow(groupInfo->mUsage, mSize);
   groupInfo->mUsage += mSize;
+
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, mSize);
   quotaManager->mTemporaryStorageUsage += mSize;
 }
 
 bool
 QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
 {
+  AssertNoOverflow(aOffset, aCount);
   int64_t end = aOffset + aCount;
 
   QuotaManager* quotaManager = QuotaManager::Get();
@@ -106,26 +191,32 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
     groupInfo->mGroupInfoPair->LockedGetGroupInfo(
       ComplementaryPersistenceType(groupInfo->mPersistenceType));
 
+  AssertNoUnderflow(end, mSize);
   uint64_t delta = end - mSize;
 
+  AssertNoOverflow(mOriginInfo->mUsage, delta);
   uint64_t newUsage = mOriginInfo->mUsage + delta;
 
   // Temporary storage has no limit for origin usage (there's a group and the
   // global limit though).
 
+  AssertNoOverflow(groupInfo->mUsage, delta);
   uint64_t newGroupUsage = groupInfo->mUsage + delta;
 
   uint64_t groupUsage = groupInfo->mUsage;
   if (complementaryGroupInfo) {
+    AssertNoOverflow(groupUsage, complementaryGroupInfo->mUsage);
     groupUsage += complementaryGroupInfo->mUsage;
   }
 
   // Temporary storage has a hard limit for group usage (20 % of the global
   // limit).
+  AssertNoOverflow(groupUsage, delta);
   if (groupUsage + delta > quotaManager->GetGroupLimit()) {
     return false;
   }
 
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, delta);
   uint64_t newTemporaryStorageUsage = quotaManager->mTemporaryStorageUsage +
                                       delta;
 
@@ -181,17 +272,22 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
     // We unlocked and relocked several times so we need to recompute all the
     // essential variables and recheck the group limit.
 
+    AssertNoUnderflow(end, mSize);
     delta = end - mSize;
 
+    AssertNoOverflow(mOriginInfo->mUsage, delta);
     newUsage = mOriginInfo->mUsage + delta;
 
+    AssertNoOverflow(groupInfo->mUsage, delta);
     newGroupUsage = groupInfo->mUsage + delta;
 
     groupUsage = groupInfo->mUsage;
     if (complementaryGroupInfo) {
+      AssertNoOverflow(groupUsage, complementaryGroupInfo->mUsage);
       groupUsage += complementaryGroupInfo->mUsage;
     }
 
+    AssertNoOverflow(groupUsage, delta);
     if (groupUsage + delta > quotaManager->GetGroupLimit()) {
       // Unfortunately some other thread increased the group usage in the
       // meantime and we are not below the group limit anymore.
@@ -204,6 +300,7 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
       return false;
     }
 
+    AssertNoOverflow(quotaManager->mTemporaryStorageUsage, delta);
     newTemporaryStorageUsage = quotaManager->mTemporaryStorageUsage + delta;
 
     NS_ASSERTION(newTemporaryStorageUsage <=
@@ -211,14 +308,13 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
 
     // Ok, we successfully freed enough space and the operation can continue
     // without throwing the quota error.
-
     mOriginInfo->mUsage = newUsage;
     groupInfo->mUsage = newGroupUsage;
     quotaManager->mTemporaryStorageUsage = newTemporaryStorageUsage;;
 
     // Some other thread could increase the size in the meantime, but no more
     // than this one.
-    NS_ASSERTION(mSize < end, "This shouldn't happen!");
+    MOZ_ASSERT(mSize < end);
     mSize = end;
 
     // Finally, release IO thread only objects and allow next synchronized
@@ -244,13 +340,16 @@ OriginInfo::LockedDecreaseUsage(int64_t aSize)
 {
   AssertCurrentThreadOwnsQuotaMutex();
 
+  AssertNoUnderflow(mUsage, aSize);
   mUsage -= aSize;
 
+  AssertNoUnderflow(mGroupInfo->mUsage, aSize);
   mGroupInfo->mUsage -= aSize;
 
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
+  AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, aSize);
   quotaManager->mTemporaryStorageUsage -= aSize;
 }
 
@@ -294,11 +393,13 @@ GroupInfo::LockedAddOriginInfo(OriginInfo* aOriginInfo)
                "Replacing an existing entry!");
   mOriginInfos.AppendElement(aOriginInfo);
 
+  AssertNoOverflow(mUsage, aOriginInfo->mUsage);
   mUsage += aOriginInfo->mUsage;
 
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, aOriginInfo->mUsage);
   quotaManager->mTemporaryStorageUsage += aOriginInfo->mUsage;
 }
 
@@ -309,14 +410,14 @@ GroupInfo::LockedRemoveOriginInfo(const nsACString& aOrigin)
 
   for (uint32_t index = 0; index < mOriginInfos.Length(); index++) {
     if (mOriginInfos[index]->mOrigin == aOrigin) {
-      MOZ_ASSERT(mUsage >= mOriginInfos[index]->mUsage);
+      AssertNoUnderflow(mUsage, mOriginInfos[index]->mUsage);
       mUsage -= mOriginInfos[index]->mUsage;
 
       QuotaManager* quotaManager = QuotaManager::Get();
       MOZ_ASSERT(quotaManager);
 
-      MOZ_ASSERT(quotaManager->mTemporaryStorageUsage >=
+      AssertNoUnderflow(quotaManager->mTemporaryStorageUsage,
-                 mOriginInfos[index]->mUsage);
+                        mOriginInfos[index]->mUsage);
       quotaManager->mTemporaryStorageUsage -= mOriginInfos[index]->mUsage;
 
       mOriginInfos.RemoveElementAt(index);
@@ -337,10 +438,10 @@ GroupInfo::LockedRemoveOriginInfos()
   for (uint32_t index = mOriginInfos.Length(); index > 0; index--) {
     OriginInfo* originInfo = mOriginInfos[index - 1];
 
-    MOZ_ASSERT(mUsage >= originInfo->mUsage);
+    AssertNoUnderflow(mUsage, originInfo->mUsage);
     mUsage -= originInfo->mUsage;
 
-    MOZ_ASSERT(quotaManager->mTemporaryStorageUsage >= originInfo->mUsage);
+    AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, originInfo->mUsage);
     quotaManager->mTemporaryStorageUsage -= originInfo->mUsage;
 
     mOriginInfos.RemoveElementAt(index - 1);

storage/src/TelemetryVFS.cpp

@@ -144,6 +144,198 @@ struct telemetry_file {
   sqlite3_file pReal[1];
 };
 
+const char*
+DatabasePathFromWALPath(const char *zWALName)
+{
+  /**
+   * Do some sketchy pointer arithmetic to find the parameter key. The WAL
+   * filename is in the middle of a big allocated block that contains:
+   *
+   *   - Random Values
+   *   - Main Database Path
+   *   - \0
+   *   - Multiple URI components consisting of:
+   *     - Key
+   *     - \0
+   *     - Value
+   *     - \0
+   *   - \0
+   *   - Journal Path
+   *   - \0
+   *   - WAL Path (zWALName)
+   *   - \0
+   *
+   * Because the main database path is preceded by a random value we have to be
+   * careful when trying to figure out when we should terminate this loop.
+   */
+  MOZ_ASSERT(zWALName);
+
+  nsDependentCSubstring dbPath(zWALName, strlen(zWALName));
+
+  // Chop off the "-wal" suffix.
+  NS_NAMED_LITERAL_CSTRING(kWALSuffix, "-wal");
+  MOZ_ASSERT(StringEndsWith(dbPath, kWALSuffix));
+
+  dbPath.Rebind(zWALName, dbPath.Length() - kWALSuffix.Length());
+  MOZ_ASSERT(!dbPath.IsEmpty());
+
+  // We want to scan to the end of the key/value URI pairs. Skip the preceding
+  // null and go to the last char of the journal path.
+  const char* cursor = zWALName - 2;
+
+  // Make sure we just skipped a null.
+  MOZ_ASSERT(!*(cursor + 1));
+
+  // Walk backwards over the journal path.
+  while (*cursor) {
+    cursor--;
+  }
+
+  // There should be another null here.
+  cursor--;
+  MOZ_ASSERT(!*cursor);
+
+  // Back up one more char to the last char of the previous string. It may be
+  // the database path or it may be a key/value URI pair.
+  cursor--;
+
+#ifdef DEBUG
+  {
+    // Verify that we just walked over the journal path. Account for the two
+    // nulls we just skipped.
+    const char *journalStart = cursor + 3;
+
+    nsDependentCSubstring journalPath(journalStart,
+                                      strlen(journalStart));
+
+    // Chop off the "-journal" suffix.
+    NS_NAMED_LITERAL_CSTRING(kJournalSuffix, "-journal");
+    MOZ_ASSERT(StringEndsWith(journalPath, kJournalSuffix));
+
+    journalPath.Rebind(journalStart,
+                        journalPath.Length() - kJournalSuffix.Length());
+    MOZ_ASSERT(!journalPath.IsEmpty());
+
+    // Make sure that the database name is a substring of the journal name.
+    MOZ_ASSERT(journalPath == dbPath);
+  }
+#endif
+
+  // Now we're either at the end of the key/value URI pairs or we're at the
+  // end of the database path. Carefully walk backwards one character at a
+  // time to do this safely without running past the beginning of the database
+  // path.
+  const char *const dbPathStart = dbPath.BeginReading();
+  const char *dbPathCursor = dbPath.EndReading() - 1;
+  bool isDBPath = true;
+
+  while (true) {
+    MOZ_ASSERT(*dbPathCursor, "dbPathCursor should never see a null char!");
+
+    if (isDBPath) {
+      isDBPath = dbPathStart <= dbPathCursor &&
+                 *dbPathCursor == *cursor &&
+                 *cursor;
+    }
+
+    if (!isDBPath) {
+      // This isn't the database path so it must be a value. Scan past it and
+      // the key also.
+      for (size_t stringCount = 0; stringCount < 2; stringCount++) {
+        // Scan past the string to the preceding null character.
+        while (*cursor) {
+          cursor--;
+        }
+
+        // Back up one more char to the last char of preceding string.
+        cursor--;
+      }
+
+      // Reset and start again.
+      dbPathCursor = dbPath.EndReading() - 1;
+      isDBPath = true;
+
+      continue;
+    }
+
+    MOZ_ASSERT(isDBPath);
+    MOZ_ASSERT(*cursor);
+
+    if (dbPathStart == dbPathCursor) {
+      // Found the full database path, we're all done.
+      MOZ_ASSERT(nsDependentCString(cursor) == dbPath);
+      return cursor;
+    }
+
+    // Change the cursors and go through the loop again.
+    cursor--;
+    dbPathCursor--;
+  }
+
+  MOZ_CRASH("Should never get here!");
+}
+
+already_AddRefed<QuotaObject>
+GetQuotaObjectFromNameAndParameters(const char *zName,
+                                    const char *zURIParameterKey)
+{
+  MOZ_ASSERT(zName);
+  MOZ_ASSERT(zURIParameterKey);
+
+  const char *persistenceType =
+    persistenceType = sqlite3_uri_parameter(zURIParameterKey,
+                                            "persistenceType");
+  if (!persistenceType) {
+    return nullptr;
+  }
+
+  const char *group = sqlite3_uri_parameter(zURIParameterKey, "group");
+  if (!group) {
+    NS_WARNING("SQLite URI had 'persistenceType' but not 'group'?!");
+    return nullptr;
+  }
+
+  const char *origin = sqlite3_uri_parameter(zURIParameterKey, "origin");
+  if (!origin) {
+    NS_WARNING("SQLite URI had 'persistenceType' and 'group' but not "
+               "'origin'?!");
+    return nullptr;
+  }
+
+  QuotaManager *quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  return quotaManager->GetQuotaObject(
+    PersistenceTypeFromText(nsDependentCString(persistenceType)),
+    nsDependentCString(group),
+    nsDependentCString(origin),
+    NS_ConvertUTF8toUTF16(zName));
+}
+
+void
+MaybeEstablishQuotaControl(const char *zName,
+                           telemetry_file *pFile,
+                           int flags)
+{
+  MOZ_ASSERT(pFile);
+  MOZ_ASSERT(!pFile->quotaObject);
+
+  if (!(flags & (SQLITE_OPEN_URI | SQLITE_OPEN_WAL))) {
+    return;
+  }
+
+  MOZ_ASSERT(zName);
+
+  const char *zURIParameterKey = (flags & SQLITE_OPEN_WAL) ?
+                                 DatabasePathFromWALPath(zName) :
+                                 zName;
+
+  MOZ_ASSERT(zURIParameterKey);
+
+  pFile->quotaObject =
+    GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey);
+}
+
 /*
 ** Close a telemetry_file.
 */
@@ -197,6 +389,19 @@ xWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst)
   return rc;
 }
 
+/*
+** Return the current file-size of a telemetry_file.
+*/
+int
+xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
+{
+  IOThreadAutoTimer ioTimer(IOInterposeObserver::OpStat);
+  telemetry_file *p = (telemetry_file *)pFile;
+  int rc;
+  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
+  return rc;
+}
+
 /*
 ** Truncate a telemetry_file.
 */
@@ -209,7 +414,15 @@ xTruncate(sqlite3_file *pFile, sqlite_int64 size)
   Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_TRUNCATE_MS> timer;
   rc = p->pReal->pMethods->xTruncate(p->pReal, size);
   if (rc == SQLITE_OK && p->quotaObject) {
-    p->quotaObject->UpdateSize(size);
+    // xTruncate doesn't always set the size of the file to the exact size
+    // requested (e.g. if a growth increment has been specified it will round up
+    // to the next multiple of the chunk size). Use xFileSize to see what the
+    // real size is.
+    sqlite_int64 newSize;
+    rc = xFileSize(pFile, &newSize);
+    if (rc == SQLITE_OK) {
+      p->quotaObject->UpdateSize(newSize);
+    }
   }
   return rc;
 }
@@ -225,19 +438,6 @@ xSync(sqlite3_file *pFile, int flags)
   return p->pReal->pMethods->xSync(p->pReal, flags);
 }
 
-/*
-** Return the current file-size of a telemetry_file.
-*/
-int
-xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
-{
-  IOThreadAutoTimer ioTimer(IOInterposeObserver::OpStat);
-  telemetry_file *p = (telemetry_file *)pFile;
-  int rc;
-  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
-  return rc;
-}
-
 /*
 ** Lock a telemetry_file.
 */
@@ -386,20 +586,7 @@ xOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* pFile,
   }
   p->histograms = h;
 
-  const char* persistenceType;
+  MaybeEstablishQuotaControl(zName, p, flags);
-  const char* group;
-  const char* origin;
-  if ((flags & SQLITE_OPEN_URI) &&
-      (persistenceType = sqlite3_uri_parameter(zName, "persistenceType")) &&
-      (group = sqlite3_uri_parameter(zName, "group")) &&
-      (origin = sqlite3_uri_parameter(zName, "origin"))) {
-    QuotaManager* quotaManager = QuotaManager::Get();
-    MOZ_ASSERT(quotaManager);
-
-    p->quotaObject = quotaManager->GetQuotaObject(PersistenceTypeFromText(
-      nsDependentCString(persistenceType)), nsDependentCString(group),
-      nsDependentCString(origin), NS_ConvertUTF8toUTF16(zName));
-  }
 
   rc = orig_vfs->xOpen(orig_vfs, zName, p->pReal, flags, pOutFlags);
   if( rc != SQLITE_OK )
@@ -451,7 +638,22 @@ int
 xDelete(sqlite3_vfs* vfs, const char *zName, int syncDir)
 {
   sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
-  return orig_vfs->xDelete(orig_vfs, zName, syncDir);
+  int rc;
+  nsRefPtr<QuotaObject> quotaObject;
+
+  if (StringEndsWith(nsDependentCString(zName), NS_LITERAL_CSTRING("-wal"))) {
+    const char *zURIParameterKey = DatabasePathFromWALPath(zName);
+    MOZ_ASSERT(zURIParameterKey);
+
+    quotaObject = GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey);
+  }
+
+  rc = orig_vfs->xDelete(orig_vfs, zName, syncDir);
+  if (rc == SQLITE_OK && quotaObject) {
+    quotaObject->UpdateSize(0);
+  }
+
+  return rc;
 }
 
 int