Bug 332748 Figure out how much history we can show by default (browser.history_expire_days now set to 180 days) (r=sspitzer)

browser/app/profile/firefox.js

@@ -251,6 +251,7 @@ pref("browser.history.grouping", "day");
 pref("browser.history.showSessions", false);
 pref("browser.sessionhistory.max_entries", 50);
 pref("browser.history_expire_days", 180);
+pref("browser.history_expire_visits", 20000);
 
 // handle external links
 // 0=default window, 1=current window/tab, 2=new window, 3=new tab in most recent window

toolkit/components/places/src/nsNavHistory.cpp

@@ -105,6 +105,7 @@
 // preference ID strings
 #define PREF_BRANCH_BASE                        "browser."
 #define PREF_BROWSER_HISTORY_EXPIRE_DAYS        "history_expire_days"
+#define PREF_BROWSER_HISTORY_EXPIRE_VISITS      "history_expire_visits"
 #define PREF_AUTOCOMPLETE_ONLY_TYPED            "urlbar.matchOnlyTyped"
 #define PREF_AUTOCOMPLETE_ENABLED               "urlbar.autocomplete.enabled"
 #define PREF_DB_CACHE_PERCENTAGE                "history_cache_percentage"
@@ -154,10 +155,18 @@
 // check idle timer every 5 minutes
 #define IDLE_TIMER_TIMEOUT (300 * PR_MSEC_PER_SEC)
 
-// perform vacuum every 15 mins *** CURRENTLY DISABLED ***
+// *** CURRENTLY DISABLED ***
+// Perform vacuum after 15 minutes of idle time, repeating.
 // 15 minutes = 900 seconds = 900000 milliseconds
 #define VACUUM_IDLE_TIME_IN_MSECS (900000)
 
+// Perform expiration after 5 minutes of idle time, repeating.
+// 5 minutes = 300 seconds = 300000 milliseconds
+#define EXPIRE_IDLE_TIME_IN_MSECS (300000)
+
+// Amount of items to expire at idle time.
+#define MAX_EXPIRE_RECORDS_ON_IDLE 200
+
 NS_IMPL_ADDREF(nsNavHistory)
 NS_IMPL_RELEASE(nsNavHistory)
 
@@ -256,6 +265,7 @@ nsNavHistory::nsNavHistory() : mNowValid(PR_FALSE),
                                mExpireNowTimer(nsnull),
                                mExpire(this),
                                mExpireDays(0),
+                               mExpireVisits(0),
                                mAutoCompleteOnlyTyped(PR_FALSE),
                                mBatchLevel(0),
                                mLock(nsnull),
@@ -398,6 +408,7 @@ nsNavHistory::Init()
   if (pbi) {
     pbi->AddObserver(PREF_AUTOCOMPLETE_ONLY_TYPED, this, PR_FALSE);
     pbi->AddObserver(PREF_BROWSER_HISTORY_EXPIRE_DAYS, this, PR_FALSE);
+    pbi->AddObserver(PREF_BROWSER_HISTORY_EXPIRE_VISITS, this, PR_FALSE);
   }
 
   observerService->AddObserver(this, gQuitApplicationMessage, PR_FALSE);
@@ -1369,6 +1380,7 @@ nsNavHistory::LoadPrefs()
     return NS_OK;
 
   mPrefBranch->GetIntPref(PREF_BROWSER_HISTORY_EXPIRE_DAYS, &mExpireDays);
+  mPrefBranch->GetIntPref(PREF_BROWSER_HISTORY_EXPIRE_VISITS, &mExpireVisits);
   PRBool oldCompleteOnlyTyped = mAutoCompleteOnlyTyped;
   mPrefBranch->GetBoolPref(PREF_AUTOCOMPLETE_ONLY_TYPED,
                            &mAutoCompleteOnlyTyped);
@@ -3408,6 +3420,15 @@ nsNavHistory::OnIdle()
   rv = idleService->GetIdleTime(&idleTime);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // If we've been idle for more than EXPIRE_IDLE_TIME_IN_MSECS
+  // keep the expiration engine chugging along.
+  // Note: This is done prior to a possible vacuum, to optimize space reduction
+  // in the vacuum.
+  if (idleTime > EXPIRE_IDLE_TIME_IN_MSECS) {
+    PRBool dummy;
+    (void)mExpire.ExpireItems(MAX_EXPIRE_RECORDS_ON_IDLE, &dummy);
+  }
+
   // If we've been idle for more than VACUUM_IDLE_TIME_IN_MSECS
   // perform a vacuum.
   if (idleTime > VACUUM_IDLE_TIME_IN_MSECS) {
@@ -3474,8 +3495,9 @@ nsNavHistory::Observe(nsISupports *aSubject, const char *aTopic,
     observerService->RemoveObserver(this, gQuitApplicationMessage);
   } else if (nsCRT::strcmp(aTopic, "nsPref:changed") == 0) {
     PRInt32 oldDays = mExpireDays;
+    PRInt32 oldVisits = mExpireVisits;
     LoadPrefs();
-    if (oldDays != mExpireDays)
+    if (oldDays != mExpireDays || oldVisits != mExpireVisits)
       mExpire.OnExpirationChanged();
   }
 

toolkit/components/places/src/nsNavHistory.h

@@ -600,6 +600,7 @@ protected:
   void DoneSearching();
 
   PRInt32 mExpireDays;
+  PRInt32 mExpireVisits;
 
   // in nsNavHistoryQuery.cpp
   nsresult TokensToQueries(const nsTArray<QueryKeyValuePair>& aTokens,

toolkit/components/places/src/nsNavHistoryExpire.cpp

@@ -68,6 +68,9 @@ struct nsNavHistoryExpireRecord {
 // running the query.
 #define EXPIRATION_COUNT_PER_RUN 6
 
+// Larger expiration chunk for idle time and shutdown.
+#define EXPIRATION_COUNT_PER_RUN_LARGE 50
+
 // The time in ms to wait after AddURI to try expiration of pages. Short is
 // actually better. If expiration takes an unusually long period of time, it
 // will interfere with video playback in the browser, for example. Such a blip
@@ -93,6 +96,10 @@ const PRTime EXPIRATION_POLICY_MONTHS = ((PRTime)180 * 86400 * PR_USEC_PER_SEC);
 // Expiration policy for embedded links (bug #401722)
 const PRTime EMBEDDED_LINK_LIFETIME = ((PRTime)10 * 86400 * PR_USEC_PER_SEC);
 
+// History preferences
+#define PREF_BRANCH_BASE                        "browser."
+#define PREF_BROWSER_HISTORY_EXPIRE_DAYS        "history_expire_days"
+
 // nsNavHistoryExpire::nsNavHistoryExpire
 //
 //    Warning: don't do anything with aHistory in the constructor, since
@@ -254,6 +261,8 @@ void
 nsNavHistoryExpire::OnExpirationChanged()
 {
   mNextExpirationTime = 0;
+  // kick off expiration
+  (void)OnAddURI(PR_Now());
 }
 
 
@@ -393,6 +402,9 @@ nsNavHistoryExpireRecord::nsNavHistoryExpireRecord(
 
 // nsNavHistoryExpire::FindVisits
 //
+//    Find visits to expire, up to the cap in browser.history_expire_visits
+//    then the age in browser.history_expire_days.
+//
 //    aExpireThreshold is the time at which we will delete visits before.
 //    If it is zero, we will not use a threshold and will match everything.
 //
@@ -404,33 +416,74 @@ nsNavHistoryExpire::FindVisits(PRTime aExpireThreshold, PRUint32 aNumToExpire,
                                mozIStorageConnection* aConnection,
                                nsTArray<nsNavHistoryExpireRecord>& aRecords)
 {
+  // get default browser.history_expire_days value
   nsresult rv;
+  nsCOMPtr<nsIPrefService> prefService =
+    do_GetService(NS_PREFSERVICE_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsCOMPtr<nsIPrefBranch> defaultPrefBranch;
+  rv = prefService->GetDefaultBranch(PREF_BRANCH_BASE,
+    getter_AddRefs(defaultPrefBranch));
+  NS_ENSURE_SUCCESS(rv, rv);
+  PRInt32 defaultExpireDays;
+  rv = defaultPrefBranch->GetIntPref(PREF_BROWSER_HISTORY_EXPIRE_DAYS, &defaultExpireDays);
+  NS_ENSURE_SUCCESS(rv, rv);
 
-  // get info for expiring visits, special case no threshold so there is no
-  // SQL parameter
-  nsCOMPtr<mozIStorageStatement> selectStatement;
-  nsCString sql;
-  sql.AssignLiteral("SELECT "
-      "v.id, v.place_id, v.visit_date, h.url, h.favicon_id, h.hidden, b.fk "
-      "FROM moz_historyvisits v LEFT JOIN moz_places h ON v.place_id = h.id "
-      "LEFT OUTER JOIN moz_bookmarks b on v.place_id = b.fk AND b.type = ?1 ");
-  if (aExpireThreshold != 0)
-    sql.AppendLiteral(" WHERE visit_date < ?2");
-  rv = aConnection->CreateStatement(sql, getter_AddRefs(selectStatement));
-  NS_ENSURE_SUCCESS(rv, rv);
-  rv = selectStatement->BindInt32Parameter(0, nsINavBookmarksService::TYPE_BOOKMARK);
-  NS_ENSURE_SUCCESS(rv, rv);
-  if (aExpireThreshold != 0) {
-    rv = selectStatement->BindInt64Parameter(1, aExpireThreshold);
+  // base sql
+  nsCAutoString sqlBase;
+  sqlBase.AssignLiteral(
+    "SELECT v.id, v.place_id, v.visit_date, h.url, h.favicon_id, h.hidden, b.fk "
+    "FROM moz_historyvisits v LEFT JOIN moz_places h ON v.place_id = h.id "
+    "LEFT OUTER JOIN moz_bookmarks b on v.place_id = b.fk ");
+
+  // If history_expire_days is the default then go ahead and expire up to the
+  // visit cap, else we respect the user's value, or if we're clearing history.
+  if (defaultExpireDays == mHistory->mExpireDays || !aNumToExpire) {
+    // build capped query
+    nsCOMPtr<mozIStorageStatement> visitsStatement;
+    nsCAutoString sqlVisits;
+    sqlVisits.Assign(sqlBase);
+    if (aNumToExpire) {
+      // Setting the visit cap as the OFFSET value selects the next aNumToExpire
+      // records above the cap.
+      sqlVisits.AppendLiteral("ORDER BY v.visit_date DESC LIMIT ?1 OFFSET ?2 ");
+    }
+    rv = aConnection->CreateStatement(sqlVisits, getter_AddRefs(visitsStatement));
     NS_ENSURE_SUCCESS(rv, rv);
+
+    if (aNumToExpire) {
+      rv = visitsStatement->BindInt64Parameter(0, aNumToExpire);
+      NS_ENSURE_SUCCESS(rv, rv);
+      rv = visitsStatement->BindInt32Parameter(1, mHistory->mExpireVisits);
+      NS_ENSURE_SUCCESS(rv, rv);
+    }
+
+    PRBool hasMore = PR_FALSE;
+    while (NS_SUCCEEDED(visitsStatement->ExecuteStep(&hasMore)) && hasMore) {
+      nsNavHistoryExpireRecord record(visitsStatement);
+      aRecords.AppendElement(record);
+    }
   }
 
-  PRBool hasMore = PR_FALSE;
-  while (NS_SUCCEEDED(selectStatement->ExecuteStep(&hasMore)) && hasMore &&
-         (aNumToExpire == 0 || aRecords.Length() < aNumToExpire)) {
-    nsNavHistoryExpireRecord record(selectStatement);
-    aRecords.AppendElement(record);
+  if (aExpireThreshold && aRecords.Length() < aNumToExpire) {
+    nsCOMPtr<mozIStorageStatement> selectStatement;
+    nsCAutoString sqlDate;
+    sqlDate.Assign(sqlBase);
+    sqlDate.AppendLiteral("AND visit_date < ?1 LIMIT ?2");
+    rv = aConnection->CreateStatement(sqlDate, getter_AddRefs(selectStatement));
+    NS_ENSURE_SUCCESS(rv, rv);
+    rv = selectStatement->BindInt64Parameter(0, aExpireThreshold);
+    NS_ENSURE_SUCCESS(rv, rv);
+    rv = selectStatement->BindInt32Parameter(1, aNumToExpire - aRecords.Length());
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    PRBool hasMore = PR_FALSE;
+    while (NS_SUCCEEDED(selectStatement->ExecuteStep(&hasMore)) && hasMore) {
+      nsNavHistoryExpireRecord record(selectStatement);
+      aRecords.AppendElement(record);
+    }
   }
+
   return NS_OK;
 }
 
@@ -770,15 +823,9 @@ nsNavHistoryExpire::ExpireForDegenerateRuns()
   if (mAnyEmptyRuns)
     return PR_FALSE;
 
-  // If very few URIs were added this run, or we expired more items than we
-  // added, don't worry about it
-  if (mAddCount < 10 || mAddCount < mExpiredItems)
-    return PR_FALSE;
-
-  // This run looks suspicious, try to expire up to the number of items
-  // we may have missed this session.
+  // Expire a larger chunk of runs to catch up.
   PRBool keepGoing;
-  nsresult rv = ExpireItems(mAddCount - mExpiredItems, &keepGoing);
+  nsresult rv = ExpireItems(EXPIRATION_COUNT_PER_RUN_LARGE, &keepGoing);
   if (NS_FAILED(rv))
     NS_WARNING("ExpireItems failed.");
   return PR_TRUE;

toolkit/components/places/src/nsNavHistoryExpire.h

@@ -57,6 +57,7 @@ public:
   void OnQuit();
   nsresult ClearHistory();
   void OnExpirationChanged();
+  nsresult ExpireItems(PRUint32 aNumToExpire, PRBool* aKeepGoing);
 
 protected:
 
@@ -86,7 +87,6 @@ protected:
 
   nsresult DoPartialExpiration();
 
-  nsresult ExpireItems(PRUint32 aNumToExpire, PRBool* aKeepGoing);
   nsresult ExpireAnnotations(mozIStorageConnection* aConnection);
   nsresult ExpireEmbeddedLinks(mozIStorageConnection* aConnection);
 

toolkit/components/places/tests/unit/test_expiration.js

@@ -106,14 +106,14 @@ var dbService = Cc["@mozilla.org/storage/service;1"].getService(Ci.mozIStorageSe
 var dbConnection = dbService.openDatabase(dbFile);
   
 
+var testURI = uri("http://mozilla.com");
+var testAnnoName = "tests/expiration/history";
+var testAnnoVal = "foo";
+var bookmark = bmsvc.insertBookmark(bmsvc.bookmarksRoot, testURI, bmsvc.DEFAULT_INDEX, "foo");
+var triggerURI = uri("http://foobar.com");
+
 // main
 function run_test() {
-  var testURI = uri("http://mozilla.com");
-  var testAnnoName = "tests/expiration/history";
-  var testAnnoVal = "foo";
-  var bookmark = bmsvc.insertBookmark(bmsvc.bookmarksRoot, testURI, bmsvc.DEFAULT_INDEX, "foo");
-  var triggerURI = uri("http://foobar.com");
-
   /*
   test that nsIBrowserHistory.removePagesFromHost does remove expirable annotations
   but doesn't remove bookmarks or EXPIRE_NEVER annotations.
@@ -163,25 +163,6 @@ function run_test() {
   do_check_neq(histsvc.getPageTitle(placeURI), null);
   // for bookmarked URI
   do_check_neq(histsvc.getPageTitle(bmURI), null);
-  
-
-  /*
-  test age-based history and anno expiration via the browser.history_expire_days pref.
-  steps:
-    - add a visit (at least 2 days old)
-    - set an anno on that entry
-    - set pref to 1 day
-    - call histsvc.removeAllPages()
-    - check onPageExpired for that entry
-  */
-  histsvc.addVisit(testURI, Date.now() - (86400 * 2), 0, histsvc.TRANSITION_TYPED, false, 0);
-  annosvc.setPageAnnotation(testURI, testAnnoName, testAnnoVal, 0, annosvc.EXPIRE_WITH_HISTORY);
-  var prefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch);
-  prefs.setIntPref("browser.history_expire_days", 1);
-  histsvc.removeAllPages();
-  do_check_true(observer.historyCleared);
-  do_check_eq(testURI.spec, observer.expiredURI);
-  do_check_eq(annosvc.getPageAnnotationNames(testURI, {}).length, 0);
 
   /*
   test anno expiration (expire never)
@@ -460,4 +441,95 @@ function run_test() {
     annosvc.getItemAnnotation(bookmark, testAnnoName);
     do_throw("bookmark lost a days anno that was modified 8 days ago");
   } catch(ex) {}
+
+  startIncrementalExpirationTests();
+}
+
+// incremental expiration tests
+// run async, chained
+
+function startIncrementalExpirationTests() {
+  startExpireByVisitsTest();
+}
+
+var prefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch);
+var ghist = Cc["@mozilla.org/browser/global-history;2"].getService(Ci.nsIGlobalHistory2);
+
+/*
+test visit-count-based history expiration via the browser.history_expire_visits pref.
+steps:
+  - reset observer
+  - clear history
+  - add 6 visits
+  - set browser.history_expire_visits pref to 1 visit
+  - kick off incremental expiration via addURI
+  - check onPageExpired for the older visit
+*/
+function startExpireByVisitsTest() {
+  dump("starting history_expire_visits test\n");
+  observer.expiredURI = null;
+  histsvc.removeAllPages();
+  var fillerURI = uri("http://blah.com");
+  for (var i = 0; i < 5; i++)
+    histsvc.addVisit(uri("http://filler.com/" + i), Date.now(), 0, histsvc.TRANSITION_TYPED, false, 0);
+  // 6th visit (incremental expiration does chunks of 6, see EXPIRATION_COUNT_PER_RUN)
+  // distinguish it from the other visits, and make it the oldest
+  histsvc.addVisit(testURI, Date.now() - (86400 * 2), 0, histsvc.TRANSITION_TYPED, false, 0);
+  annosvc.setPageAnnotation(testURI, testAnnoName, testAnnoVal, 0, annosvc.EXPIRE_WITH_HISTORY);
+  prefs.setIntPref("browser.history_expire_visits", 1);
+  // addURI triggers incremental expiration, and is also the 7th visit
+  ghist.addURI(uri("http://fizz.com"), false, true, triggerURI); 
+  do_test_pending();
+  do_timeout(3600, "checkExpireByVisitsTest();"); // incremental expiration timer is 3500, see PARTIAL_EXPIRATION_TIMEOUT 
+}
+
+function checkExpireByVisitsTest() {
+  try {
+    do_check_eq(testURI.spec, observer.expiredURI);
+    do_check_eq(annosvc.getPageAnnotationNames(testURI, {}).length, 0);
+    do_check_eq(histsvc.getPageTitle(uri("http://fizz.com")), "fizz.com");
+  } catch(ex) {}
+  dump("done history_expire_visits test\n");
+  startExpireByDaysTest();
+}
+
+
+/*
+test fallback to age-based history expiration via the browser.history_expire_days pref.
+
+ie: assuming there are less than browser.history_expire_visits visits, expire any visits
+over browser.history_expire_days old.
+
+steps:
+  - reset observer
+  - clear history
+  - add 6 visits, each 2 days old
+  - browser.history_expire_visits is at 1
+  - set browser.history_expire_days pref to 1 day
+  - kick off incremental expiration
+  - check onPageExpired for the older visit
+*/
+function startExpireByDaysTest() {
+  dump("starting history_expire_days test\n");
+  observer.expiredURI = null;
+  histsvc.removeAllPages();
+  histsvc.addVisit(uri("http://blah.com"), Date.now() - (86400 * 2), 0, histsvc.TRANSITION_TYPED, false, 0);
+  histsvc.addVisit(uri("http://bleh.com"), Date.now() - (86400 * 2), 0, histsvc.TRANSITION_TYPED, false, 0);
+  histsvc.addVisit(testURI, Date.now() - (86400 * 2), 0, histsvc.TRANSITION_TYPED, false, 0);
+  annosvc.setPageAnnotation(testURI, testAnnoName, testAnnoVal, 0, annosvc.EXPIRE_WITH_HISTORY);
+  var prefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch);
+  // history_expire_visits is set to 1 already
+  // setting history_expire_days to 1 will cause all of our 2 day-old visits to expire
+  prefs.setIntPref("browser.history_expire_days", 1);
+  ghist.addURI(testURI, false, true, triggerURI); 
+  do_timeout(3600, "checkExpireByDaysTest();"); // incremental expiration timer is 3500
+}
+
+function checkExpireByDaysTest() {
+  try {
+    do_check_eq(testURI.spec, observer.expiredURI);
+    do_check_eq(annosvc.getPageAnnotationNames(testURI, {}).length, 0);
+  } catch(ex) {}
+  dump("done history_expire_days test\n");
+  do_test_finished();
 }