Bug 945779 - Use transactions to batch up Seer I/O. r=honzab

netwerk/base/public/nsINetworkSeer.idl

@@ -17,7 +17,7 @@ typedef unsigned long SeerLearnReason;
  *                  predictive actions upon future visits.
  *                  NOTE: nsINetworkSeer should only be used on the main thread
  */
-[scriptable, uuid(884a39a0-a3ed-4855-826a-fabb73ae878d)]
+[scriptable, uuid(25e323b6-99e0-4274-b5b3-1a9eb56e28ac)]
 interface nsINetworkSeer : nsISupports
 {
   /**
@@ -121,6 +121,12 @@ interface nsINetworkSeer : nsISupports
    * after this completes will start from a blank slate.
    */
   void reset();
+
+  /**
+   * @deprecated THIS API IS FOR TESTING ONLY. IF YOU DON'T KNOW WHAT IT DOES,
+   * DON'T USE IT
+   */
+  void prepareForDnsTest(in long long timestamp, in string uri);
 };
 
 %{C++

netwerk/base/src/Seer.cpp

@@ -21,6 +21,7 @@
 #include "nsIPrefBranch.h"
 #include "nsIPrefService.h"
 #include "nsISpeculativeConnect.h"
+#include "nsITimer.h"
 #include "nsIURI.h"
 #include "nsNetUtil.h"
 #include "nsServiceManagerUtils.h"
@@ -290,6 +291,41 @@ Seer::RemoveObserver()
   }
 }
 
+static const uint32_t COMMIT_TIMER_DELTA_MS = 5 * 1000;
+
+class SeerCommitTimerInitEvent : public nsRunnable
+{
+public:
+  NS_IMETHOD Run() MOZ_OVERRIDE
+  {
+    nsresult rv = NS_OK;
+
+    if (!gSeer->mCommitTimer) {
+      gSeer->mCommitTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv);
+    } else {
+      gSeer->mCommitTimer->Cancel();
+    }
+    if (NS_SUCCEEDED(rv)) {
+      gSeer->mCommitTimer->Init(gSeer, COMMIT_TIMER_DELTA_MS,
+                                nsITimer::TYPE_ONE_SHOT);
+    }
+
+    return NS_OK;
+  }
+};
+
+class SeerNewTransactionEvent : public nsRunnable
+{
+  NS_IMETHODIMP Run() MOZ_OVERRIDE
+  {
+    gSeer->CommitTransaction();
+    gSeer->BeginTransaction();
+    nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
+    NS_DispatchToMainThread(event);
+    return NS_OK;
+  }
+};
+
 NS_IMETHODIMP
 Seer::Observe(nsISupports *subject, const char *topic,
               const char16_t *data_unicode)
@@ -298,6 +334,9 @@ Seer::Observe(nsISupports *subject, const char *topic,
 
   if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {
     gSeer->Shutdown();
+  } else if (!strcmp(NS_TIMER_CALLBACK_TOPIC, topic)) {
+    nsRefPtr<SeerNewTransactionEvent> event = new SeerNewTransactionEvent();
+    gSeer->mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
   }
 
   return rv;
@@ -467,6 +506,8 @@ Seer::EnsureInitStorage()
   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous = OFF;"));
   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA foreign_keys = ON;"));
 
+  BeginTransaction();
+
   // A table to make sure we're working with the database layout we expect
   rv = mDB->ExecuteSimpleSQL(
       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_seer_version (\n"
@@ -703,6 +744,12 @@ Seer::EnsureInitStorage()
                          "ON moz_redirects (id);"));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  CommitTransaction();
+  BeginTransaction();
+
+  nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
+  NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+
   return NS_OK;
 }
 
@@ -737,6 +784,9 @@ public:
   {
     MOZ_ASSERT(!NS_IsMainThread(), "Shutting down DB on main thread");
 
+    // Ensure everything is written to disk before we shut down the db
+    gSeer->CommitTransaction();
+
     gSeer->mStatements.FinalizeStatements();
     gSeer->mDB->Close();
     gSeer->mDB = nullptr;
@@ -765,6 +815,10 @@ Seer::Shutdown()
 
   mInitialized = false;
 
+  if (mCommitTimer) {
+    mCommitTimer->Cancel();
+  }
+
   if (mIOThread) {
     nsCOMPtr<nsIThread> ioThread;
     mIOThread.swap(ioThread);
@@ -2152,13 +2206,17 @@ Seer::ResetInternal()
   nsresult rv = EnsureInitStorage();
   RETURN_IF_FAILED(rv);
 
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups;"));
 
   // These cascade to moz_subresources and moz_subhosts, respectively.
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts;"));
+
+  // Go ahead and ensure this is flushed to disk
+  CommitTransaction();
+  BeginTransaction();
 }
 
 // Called on the main thread to clear out all our knowledge. Tabula Rasa FTW!
@@ -2176,6 +2234,68 @@ Seer::Reset()
   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
 }
 
+#ifdef SEER_TESTS
+class SeerPrepareForDnsTestEvent : public nsRunnable
+{
+public:
+  SeerPrepareForDnsTestEvent(int64_t timestamp, const char *uri)
+    :mTimestamp(timestamp)
+    ,mUri(uri)
+  { }
+
+  NS_IMETHOD Run() MOZ_OVERRIDE
+  {
+    MOZ_ASSERT(!NS_IsMainThread(), "Preparing for DNS Test on main thread!");
+    gSeer->PrepareForDnsTestInternal(mTimestamp, mUri);
+    return NS_OK;
+  }
+
+private:
+  int64_t mTimestamp;
+  nsAutoCString mUri;
+};
+
+void
+Seer::PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri)
+{
+  nsCOMPtr<mozIStorageStatement> update = mStatements.GetCachedStatement(
+      NS_LITERAL_CSTRING("UPDATE moz_subresources SET last_hit = :timestamp, "
+                          "hits = 2 WHERE uri = :uri;"));
+  if (!update) {
+    return;
+  }
+  mozStorageStatementScoper scopedUpdate(update);
+
+  nsresult rv = update->BindInt64ByName(NS_LITERAL_CSTRING("timestamp"),
+                                        timestamp);
+  RETURN_IF_FAILED(rv);
+
+  rv = update->BindUTF8StringByName(NS_LITERAL_CSTRING("uri"), uri);
+  RETURN_IF_FAILED(rv);
+
+  update->Execute();
+}
+#endif
+
+NS_IMETHODIMP
+Seer::PrepareForDnsTest(int64_t timestamp, const char *uri)
+{
+#ifdef SEER_TESTS
+  MOZ_ASSERT(NS_IsMainThread(),
+             "Seer interface methods must be called on the main thread");
+
+  if (!mInitialized) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  nsRefPtr<SeerPrepareForDnsTestEvent> event =
+    new SeerPrepareForDnsTestEvent(timestamp, uri);
+  return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
+#else
+  return NS_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
 // Helper functions to make using the seer easier from native code
 
 static nsresult

netwerk/base/src/Seer.h

@@ -22,6 +22,7 @@
 class nsIDNSService;
 class nsINetworkSeerVerifier;
 class nsIThread;
+class nsITimer;
 
 class mozIStorageConnection;
 class mozIStorageService;
@@ -61,6 +62,8 @@ private:
   friend class SeerResetEvent;
   friend class SeerPredictionRunner;
   friend class SeerDBShutdownRunner;
+  friend class SeerCommitTimerInitEvent;
+  friend class SeerNewTransactionEvent;
 
   void CheckForAndDeleteOldDBFile();
   nsresult EnsureInitStorage();
@@ -152,6 +155,16 @@ private:
 
   void ResetInternal();
 
+  void BeginTransaction()
+  {
+    mDB->BeginTransaction();
+  }
+
+  void CommitTransaction()
+  {
+    mDB->CommitTransaction();
+  }
+
   // Observer-related stuff
   nsresult InstallObserver();
   void RemoveObserver();
@@ -200,6 +213,13 @@ private:
   nsAutoPtr<SeerTelemetryAccumulators> mAccumulators;
 
   nsRefPtr<SeerDNSListener> mDNSListener;
+
+  nsCOMPtr<nsITimer> mCommitTimer;
+
+#ifdef SEER_TESTS
+  friend class SeerPrepareForDnsTestEvent;
+  void PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri);
+#endif
 };
 
 } // ::mozilla::net

netwerk/base/src/moz.build

@@ -129,3 +129,6 @@ if CONFIG['MOZ_ENABLE_QTNETWORK']:
     LOCAL_INCLUDES += [
         '/netwerk/system/qt',
     ]
+
+if CONFIG['ENABLE_TESTS']:
+    DEFINES['SEER_TESTS'] = True

netwerk/test/unit/test_seer.js

@@ -216,14 +216,7 @@ DnsContinueVerifier.prototype = {
     // x1000 on the Date object value.
     var tstamp = (new Date().valueOf() * 1000) - (10 * 86400 * 1000000);
 
-    var dbfile = FileUtils.getFile("ProfD", ["netpredictions.sqlite"]);
-    var dbconn = Services.storage.openDatabase(dbfile);
-    // We also need to update hits, since the toplevel has been "loaded" a
-    // second time (from the prediction that kicked off this callback) to ensure
-    // that the seer will try to do anything for this subresource.
-    var stmt = "UPDATE moz_subresources SET last_hit = " + tstamp + ", hits = 2 WHERE uri = '" + this.subresource + "';";
-    dbconn.executeSimpleSQL(stmt);
-    dbconn.close();
+    seer.prepareForDnsTest(tstamp, this.subresource);
 
     var verifier = new Verifier("dns", [], this.preresolves);
     seer.predict(this.tluri, null, seer.PREDICT_LOAD, load_context, verifier);