Bug 416330 - Suboptimal SQLite page size r=sdwilsh

db/sqlite3/src/Makefile.in

@@ -100,6 +100,7 @@ CSRCS = \
 # don't have to vacuum to make sure the data is not visible in the file.
 # -DSQLITE_ENABLE_FTS3=1 enables the full-text index module.
 # -DSQLITE_CORE=1 statically links that module into the SQLite library.
+# -DSQLITE_DEFAULT_PAGE_SIZE=32768 increases the page size from 1k, see bug 416330.
 # Note: Be sure to update the configure.in checks when these change!
 DEFINES = \
   -DSQLITE_SECURE_DELETE=1 \
@@ -107,6 +108,7 @@ DEFINES = \
   -DSQLITE_CORE=1 \
   -DSQLITE_ENABLE_FTS3=1 \
   -DSQLITE_ENABLE_UNLOCK_NOTIFY=1 \
+  -DSQLITE_DEFAULT_PAGE_SIZE=32768 \
   $(NULL)
 
 # -DSQLITE_ENABLE_LOCKING_STYLE=1 to help with AFP folders

storage/src/mozStorageConnection.cpp

@@ -393,6 +393,14 @@ Connection::initialize(nsIFile *aDatabaseFile)
   PR_LOG(gStorageLog, PR_LOG_NOTICE, ("Opening connection to '%s' (%p)",
                                       leafName.get(), this));
 #endif
+  // Switch db to preferred page size in case the user vacuums.
+  sqlite3_stmt *stmt;
+  srv = prepareStmt(mDBConn, NS_LITERAL_CSTRING("PRAGMA page_size = 32768"),
+                    &stmt);
+  if (srv == SQLITE_OK) {
+    (void)stepStmt(stmt);
+    (void)::sqlite3_finalize(stmt);
+  }
 
   // Register our built-in SQL functions.
   srv = registerFunctions(mDBConn);
@@ -412,7 +420,6 @@ Connection::initialize(nsIFile *aDatabaseFile)
 
   // Execute a dummy statement to force the db open, and to verify if it is
   // valid or not.
-  sqlite3_stmt *stmt;
   srv = prepareStmt(mDBConn, NS_LITERAL_CSTRING("SELECT * FROM sqlite_master"),
                     &stmt);
   if (srv == SQLITE_OK) {

storage/test/unit/test_page_size_is_32k.js

@@ -0,0 +1,28 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+// This file tests that dbs are using 32k pagesize
+
+function check_size(db)
+{
+  var stmt = db.createStatement("PRAGMA page_size");
+  stmt.executeStep();
+  const expected_block_size = 32768; // 32K
+  do_check_eq(stmt.getInt32(0), expected_block_size);
+  stmt.finalize();
+}
+
+function new_file(name)
+{
+  var file = dirSvc.get("ProfD", Ci.nsIFile);
+  file.append(name + ".sqlite");
+  do_check_false(file.exists());
+}
+
+function run_test()
+{
+  check_size(getDatabase(new_file("shared32k.sqlite")));
+  check_size(getService().openUnsharedDatabase(new_file("unshared32k.sqlite")));
+}
+

toolkit/components/places/src/nsNavHistory.cpp

@@ -140,13 +140,6 @@ using namespace mozilla::places;
 // corresponding migrateVxx method below.
 #define DATABASE_SCHEMA_VERSION 10
 
-// We set the default database page size to be larger. sqlite's default is 1K.
-// This gives good performance when many small parts of the file have to be
-// loaded for each statement. Because we try to keep large chunks of the file
-// in memory, a larger page size should give better I/O performance. 32K is
-// sqlite's default max page size.
-#define DATABASE_PAGE_SIZE 4096
-
 // Filename of the database.
 #define DATABASE_FILENAME NS_LITERAL_STRING("places.sqlite")
 
@@ -634,37 +627,23 @@ nsNavHistory::InitDBFile(PRBool aForceInit)
 nsresult
 nsNavHistory::InitDB()
 {
-  PRInt32 pageSize = DATABASE_PAGE_SIZE;
-
   // Get the database schema version.
   PRInt32 currentSchemaVersion = 0;
   nsresult rv = mDBConn->GetSchemaVersion(&currentSchemaVersion);
   NS_ENSURE_SUCCESS(rv, rv);
-  bool databaseInitialized = (currentSchemaVersion > 0);
 
-  if (!databaseInitialized) {
-    // First of all we must set page_size since it will only have effect on
-    // empty files.  For existing databases we could get a different page size,
-    // trying to change it would be uneffective.
-    // See bug 401985 for details.
-    nsCAutoString pageSizePragma("PRAGMA page_size = ");
-    pageSizePragma.AppendInt(pageSize);
-    rv = mDBConn->ExecuteSimpleSQL(pageSizePragma);
-    NS_ENSURE_SUCCESS(rv, rv);
-  }
-  else {
-    // Get the page size.  This may be different than the default if the
-    // database file already existed with a different page size.
-    nsCOMPtr<mozIStorageStatement> statement;
-    rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"),
-                                  getter_AddRefs(statement));
-    NS_ENSURE_SUCCESS(rv, rv);
+  // Get the page size.  This may be different than the default if the
+  // database file already existed with a different page size.
+  nsCOMPtr<mozIStorageStatement> statement;
+  rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"),
+                                getter_AddRefs(statement));
+  NS_ENSURE_SUCCESS(rv, rv);
 
-    PRBool hasResult;
-    rv = statement->ExecuteStep(&hasResult);
-    NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
-    pageSize = statement->AsInt32(0);
-  }
+  PRBool hasResult;
+  mozStorageStatementScoper scoper(statement);
+  rv = statement->ExecuteStep(&hasResult);
+  NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
+  PRInt32 pageSize = statement->AsInt32(0);
 
   // Ensure that temp tables are held in memory, not on disk.  We use temp
   // tables mainly for fsync and I/O reduction.
@@ -726,6 +705,7 @@ nsNavHistory::InitDB()
   rv = nsAnnotationService::InitTables(mDBConn);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  bool databaseInitialized = (currentSchemaVersion > 0);
   if (!databaseInitialized) {
     // This is the first run, so we set schema version to the latest one, since
     // we don't need to migrate anything.  We will create tables from scratch.

toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp

@@ -173,8 +173,6 @@ static const PRLogModuleInfo *gUrlClassifierDbServiceLog = nsnull;
 #define UPDATE_DELAY_TIME           "urlclassifier.updatetime"
 #define UPDATE_DELAY_TIME_DEFAULT   60
 
-#define PAGE_SIZE 4096
-
 class nsUrlClassifierDBServiceWorker;
 
 // Singleton instance.
@@ -1223,6 +1221,7 @@ private:
   nsCOMPtr<mozIStorageStatement> mGetTableIdStatement;
   nsCOMPtr<mozIStorageStatement> mGetTableNameStatement;
   nsCOMPtr<mozIStorageStatement> mInsertTableIdStatement;
+  nsCOMPtr<mozIStorageStatement> mGetPageSizeStatement;
 
   // Stores the last time a given table was updated.
   nsDataHashtable<nsCStringHashKey, PRInt64> mTableFreshness;
@@ -3164,7 +3163,13 @@ nsUrlClassifierDBServiceWorker::SetupUpdate()
   NS_ENSURE_SUCCESS(rv, rv);
 
   if (gUpdateCacheSize > 0) {
-    PRUint32 cachePages = gUpdateCacheSize / PAGE_SIZE;
+    PRBool hasResult;
+    rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
+    PRUint32 pageSize = mGetTableIdStatement->AsInt32(0);
+    PRUint32 cachePages = gUpdateCacheSize / pageSize;
     nsCAutoString cacheSizePragma("PRAGMA cache_size=");
     cacheSizePragma.AppendInt(cachePages);
     rv = mConnection->ExecuteSimpleSQL(cacheSizePragma);
@@ -3413,11 +3418,6 @@ nsUrlClassifierDBServiceWorker::OpenDb()
     }
   }
 
-  nsCAutoString cacheSizePragma("PRAGMA page_size=");
-  cacheSizePragma.AppendInt(PAGE_SIZE);
-  rv = connection->ExecuteSimpleSQL(cacheSizePragma);
-  NS_ENSURE_SUCCESS(rv, rv);
-
   rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -3475,6 +3475,11 @@ nsUrlClassifierDBServiceWorker::OpenDb()
      getter_AddRefs(mInsertTableIdStatement));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  rv = connection->CreateStatement
+    (NS_LITERAL_CSTRING("PRAGMA page_size"),
+     getter_AddRefs(mGetPageSizeStatement));
+  NS_ENSURE_SUCCESS(rv, rv);
+
   mConnection = connection;
 
   mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);