Bug 674210 - Reduce places.sqlite cache size and reorganize history expiration around the new value.

r=dietrich
This commit is contained in:
Marco Bonardo 2011-08-30 16:23:59 +02:00
parent 2e49ad6665
commit a6322ce51a
13 changed files with 148 additions and 112 deletions

View File

@ -798,12 +798,6 @@ pref("accessibility.blockautorefresh", false);
// Whether history is enabled or not.
pref("places.history.enabled", true);
// The percentage of system memory that the Places database can use. Out of the
// allowed cache size it will at most use the size of the database file.
// Changes to this value are effective after an application restart.
// Acceptable values are between 0 and 50.
pref("places.database.cache_to_memory_percentage", 6);
// the (maximum) number of the recent visits to sample
// when calculating frecency
pref("places.frecency.numVisits", 10);

View File

@ -92,8 +92,9 @@ function run_test()
getService(Ci.nsINavHistoryService);
// Add the download to places
// Add the visit in the past to circumvent possible VM timing bugs
let yesterday = Date.now() - 24 * 60 * 60 * 1000;
histsvc.addVisit(theURI, yesterday * 1000, null,
// Go back by 8 days, since expiration ignores history in the last 7 days.
let expirableTime = Date.now() - 8 * 24 * 60 * 60 * 1000;
histsvc.addVisit(theURI, expirableTime * 1000, null,
histsvc.TRANSITION_DOWNLOAD, false, 0);
// Get the download manager as history observer and batch expirations

View File

@ -128,14 +128,27 @@ using namespace mozilla::places;
#define PREF_FRECENCY_UNVISITED_BOOKMARK_BONUS "frecency.unvisitedBookmarkBonus"
#define PREF_FRECENCY_UNVISITED_TYPED_BONUS "frecency.unvisitedTypedBonus"
#define PREF_CACHE_TO_MEMORY_PERCENTAGE "database.cache_to_memory_percentage"
#define PREF_FORCE_DATABASE_REPLACEMENT "database.replaceOnStartup"
// Default integer value for PREF_CACHE_TO_MEMORY_PERCENTAGE.
// This is 6% of machine memory, giving 15MB for a user with 256MB of memory.
// Out of this cache, SQLite will use at most the size of the database file.
#define DATABASE_DEFAULT_CACHE_TO_MEMORY_PERCENTAGE 6
// To calculate the cache size we take into account the available physical
// memory and the current database size. This is the percentage of memory
// we reserve for the former case.
#define DATABASE_CACHE_TO_MEMORY_PERC 2
// The minimum size of the cache. We should never work without a cache, since
// that would badly hurt WAL journaling mode.
#define DATABASE_CACHE_MIN_BYTES (PRUint64)5242880 // 5MiB
// We calculate an optimal database size, based on hardware specs. This
// pertains more to expiration, but the code is pretty much the same used for
// cache_size, so it's here to reduce code duplication.
// This percentage of disk size is used to protect against calculating a too
// large size on disks with tiny quota or available space.
#define DATABASE_TO_DISK_PERC 2
// Maximum size of the optimal database. High-end hardware has plenty of
// memory and disk space, but performances don't grow linearly.
#define DATABASE_MAX_SIZE (PRInt64)167772160 // 160MiB
// Used to share the calculated optimal database size with other components.
#define PREF_OPTIMAL_DATABASE_SIZE "history.expiration.transient_optimal_database_size"
// If the physical memory size is not available, use MEMSIZE_FALLBACK_BYTES
// instead. Must stay in sync with the code in nsPlacesExpiration.js.
@ -681,16 +694,19 @@ nsNavHistory::SetJournalMode(enum JournalMode aJournalMode)
nsresult
nsNavHistory::InitDB()
{
// WARNING: any statement executed before setting the journal mode must be
// finalized, since SQLite doesn't allow changing the journal mode if there
// is any outstanding statement.
{
// Get the page size. This may be different than the default if the
// database file already existed with a different page size.
nsCOMPtr<mozIStorageStatement> statement;
nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"),
getter_AddRefs(statement));
nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
"PRAGMA page_size"
), getter_AddRefs(statement));
NS_ENSURE_SUCCESS(rv, rv);
PRBool hasResult;
mozStorageStatementScoper scoper(statement);
PRBool hasResult = PR_FALSE;
rv = statement->ExecuteStep(&hasResult);
NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
rv = statement->GetInt32(0, &mDBPageSize);
@ -703,29 +719,67 @@ nsNavHistory::InitDB()
"PRAGMA temp_store = MEMORY"));
NS_ENSURE_SUCCESS(rv, rv);
// Compute the size of the database cache using the device's memory size.
// We want to work with a cache that is at a maximum half of the database
// size. We also want it to respect the available memory size.
// Calculate memory size, fallback to a meaningful value if it fails.
PRUint64 memSizeBytes = PR_GetPhysicalMemorySize();
if (memSizeBytes == 0) {
memSizeBytes = MEMSIZE_FALLBACK_BYTES;
}
PRUint64 cacheSize = memSizeBytes * DATABASE_CACHE_TO_MEMORY_PERC / 100;
// Calculate an optimal database size for expiration purposes.
// We usually want to work with a cache that is half the database size.
// Limit the size to avoid extreme values on high-end hardware.
PRInt64 optimalDatabaseSize = NS_MIN(static_cast<PRInt64>(cacheSize) * 2,
DATABASE_MAX_SIZE);
// Protect against a full disk or tiny quota.
PRInt64 diskAvailableBytes = 0;
nsCOMPtr<nsILocalFile> localDB = do_QueryInterface(mDBFile);
if (localDB &&
NS_SUCCEEDED(localDB->GetDiskSpaceAvailable(&diskAvailableBytes)) &&
diskAvailableBytes > 0) {
optimalDatabaseSize = NS_MIN(optimalDatabaseSize,
diskAvailableBytes * DATABASE_TO_DISK_PERC / 100);
}
// Share the calculated size if it's meaningful.
if (optimalDatabaseSize < PR_INT32_MAX) {
(void)mPrefBranch->SetIntPref(PREF_OPTIMAL_DATABASE_SIZE,
static_cast<PRInt32>(optimalDatabaseSize));
}
// Get the current database size. Due to chunked growth we have to use
// page_count to evaluate it.
PRUint64 databaseSizeBytes = 0;
{
nsCOMPtr<mozIStorageStatement> statement;
nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
"PRAGMA page_count"
), getter_AddRefs(statement));
NS_ENSURE_SUCCESS(rv, rv);
PRBool hasResult = PR_FALSE;
rv = statement->ExecuteStep(&hasResult);
NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
PRInt32 pageCount = 0;
rv = statement->GetInt32(0, &pageCount);
NS_ENSURE_SUCCESS(rv, rv);
databaseSizeBytes = pageCount * mDBPageSize;
}
// Set cache to a maximum of half the database size.
cacheSize = NS_MIN(cacheSize, databaseSizeBytes / 2);
// Ensure we never work without a minimum cache.
cacheSize = NS_MAX(cacheSize, DATABASE_CACHE_MIN_BYTES);
// Set the number of cached pages.
// We don't use PRAGMA default_cache_size, since the database could be moved
// among different devices and the value would adapt accordingly.
PRInt32 cachePercentage;
if (NS_FAILED(mPrefBranch->GetIntPref(PREF_CACHE_TO_MEMORY_PERCENTAGE,
&cachePercentage)))
cachePercentage = DATABASE_DEFAULT_CACHE_TO_MEMORY_PERCENTAGE;
// Sanity checks, we allow values between 0 (disable cache) and 50%.
if (cachePercentage > 50)
cachePercentage = 50;
if (cachePercentage < 0)
cachePercentage = 0;
static PRUint64 physMem = PR_GetPhysicalMemorySize();
if (physMem == 0)
physMem = MEMSIZE_FALLBACK_BYTES;
PRUint64 cacheSize = physMem * cachePercentage / 100;
// Compute number of cached pages, this will be our cache size.
PRUint64 cachePages = cacheSize / mDBPageSize;
nsCAutoString cacheSizePragma("PRAGMA cache_size = ");
cacheSizePragma.AppendInt(cachePages);
cacheSizePragma.AppendInt(cacheSize / mDBPageSize);
rv = mDBConn->ExecuteSimpleSQL(cacheSizePragma);
NS_ENSURE_SUCCESS(rv, rv);
@ -4130,9 +4184,10 @@ nsNavHistory::PreparePlacesForVisitsDelete(const nsCString& aPlaceIdsQueryString
// -visit_count, as we use that value in our "on idle" query
// to figure out which places to recalculate frecency first.
// Pay attention to not set frecency = 0 if visit_count = 0
// TODO (bug 487809): we don't need anymore to set frecency here.
nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
"UPDATE moz_places "
"SET frecency = -MAX(visit_count, 1) "
"SET frecency = -(visit_count + 1) "
"WHERE id IN ( "
"SELECT h.id "
"FROM moz_places h "
@ -4563,8 +4618,9 @@ nsNavHistory::RemoveAllPages()
// Note, we set frecency to -visit_count since we use that value in our
// idle query to figure out which places to recalcuate frecency first.
// We must do this before deleting visits.
// TODO (bug 487809): we don't need anymore to set frecency here.
nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
"UPDATE moz_places SET frecency = -MAX(visit_count, 1) "
"UPDATE moz_places SET frecency = -(visit_count + 1) "
"WHERE id IN(SELECT b.fk FROM moz_bookmarks b WHERE b.fk NOTNULL)"));
NS_ENSURE_SUCCESS(rv, rv);

View File

@ -109,16 +109,10 @@ const PREF_READONLY_CALCULATED_MAX_URIS = "transient_current_max_pages";
const PREF_INTERVAL_SECONDS = "interval_seconds";
const PREF_INTERVAL_SECONDS_NOTSET = 3 * 60;
// The percentage of system memory we will use for the database's cache.
// Use the same value set in nsNavHistory.cpp. We use the size of the cache to
// evaluate how many pages we can store before going over it.
const PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE =
"places.history.cache_per_memory_percentage";
const PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE_NOTSET = 6;
// Minimum number of unique URIs to retain. This is used when system-info
// returns bogus values.
const MIN_URIS = 1000;
// An optimal database size calculated by history. Used to evaluate a limit
// to the number of pages we may retain before hitting performance issues.
const PREF_OPTIMAL_DATABASE_SIZE = "transient_optimal_database_size";
const PREF_OPTIMAL_DATABASE_SIZE_NOTSET = 167772160; // 160MiB
// Max number of entries to expire at each expiration step.
// This value is globally used for different kind of data we expire, can be
@ -141,14 +135,10 @@ const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3;
// This is the average size in bytes of an URI entry in the database.
// Magic numbers are determined through analysis of the distribution of a ratio
// between number of unique URIs and database size among our users. We use a
// more pessimistic ratio on single cores, since we handle some stuff in a
// separate thread.
// between number of unique URIs and database size among our users.
// Based on these values we evaluate how many unique URIs we can handle before
// going over the database maximum cache size. If we are over the maximum
// number of entries, we will expire.
const URIENTRY_AVG_SIZE_MIN = 2000;
const URIENTRY_AVG_SIZE_MAX = 3000;
// starting expiring some.
const URIENTRY_AVG_SIZE = 1600;
// Seconds of idle time before starting a larger expiration step.
// Notice during idle we stop the expiration timer since we don't want to hurt
@ -210,6 +200,8 @@ const EXPIRATION_QUERIES = {
// Finds visits to be expired. Will return nothing if we are not over the
// unique URIs limit.
// This explicitly excludes any visits added in the last 7 days, to protect
// users with thousands of bookmarks from constantly losing history.
QUERY_FIND_VISITS_TO_EXPIRE: {
sql: "INSERT INTO expiration_notify "
+ "(v_id, url, guid, visit_date, expected_results) "
@ -217,6 +209,7 @@ const EXPIRATION_QUERIES = {
+ "FROM moz_historyvisits v "
+ "JOIN moz_places h ON h.id = v.place_id "
+ "WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris "
+ "AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 "
+ "ORDER BY v.visit_date ASC "
+ "LIMIT :limit_visits",
actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE |
@ -235,6 +228,10 @@ const EXPIRATION_QUERIES = {
// Finds orphan URIs in the database.
// Notice we won't notify single removed URIs on removeAllPages, so we don't
// run this query in such a case, but just delete URIs.
// This could run in the middle of adding a visit or bookmark to a new page.
// In such a case since it is async, could end up expiring the orphan page
// before it actually gets the new visit or bookmark.
// Thus, since new pages get frecency -1, we filter on that.
QUERY_FIND_URIS_TO_EXPIRE: {
sql: "INSERT INTO expiration_notify "
+ "(p_id, url, guid, visit_date, expected_results) "
@ -244,7 +241,7 @@ const EXPIRATION_QUERIES = {
+ "LEFT JOIN moz_bookmarks b ON h.id = b.fk "
+ "WHERE v.id IS NULL "
+ "AND b.id IS NULL "
+ "AND h.ROWID <> IFNULL(:null_skips_last, (SELECT MAX(ROWID) FROM moz_places)) "
+ "AND frecency <> -1 "
+ "LIMIT :limit_uris",
actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
ACTION.IDLE | ACTION.DEBUG
@ -733,7 +730,8 @@ nsPlacesExpiration.prototype = {
_isIdleObserver: false,
_expireOnIdle: false,
set expireOnIdle(aExpireOnIdle) {
// Observe idle regardless, since we want to stop timed expiration.
// Observe idle regardless aExpireOnIdle, since we always want to stop
// timed expiration on idle, to preserve mobile battery life.
if (!this._isIdleObserver && !this._shuttingDown) {
this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS);
this._isIdleObserver = true;
@ -764,38 +762,15 @@ nsPlacesExpiration.prototype = {
catch(e) {}
if (this._urisLimit < 0) {
// If physical memory size is not available, use MEMSIZE_FALLBACK_BYTES
// instead. Must stay in sync with the code in nsNavHistory.cpp.
const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 M
// The preference did not exist or has a negative value, so we calculate a
// limit based on hardware.
let memsize = this._sys.getProperty("memsize"); // Memory size in bytes.
if (memsize <= 0)
memsize = MEMSIZE_FALLBACK_BYTES;
let cpucount = this._sys.getProperty("cpucount"); // CPU count.
const AVG_SIZE_PER_URIENTRY = cpucount > 1 ? URIENTRY_AVG_SIZE_MIN
: URIENTRY_AVG_SIZE_MAX;
// We will try to live inside the database cache size, since working out
// of it can be really slow.
let cache_percentage = PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE_NOTSET;
// The preference did not exist or has a negative value.
// Calculate the number of unique places that may fit an optimal database
// size on this hardware. If there are more than these unique pages,
// some will be expired.
let optimalDatabaseSize = PREF_OPTIMAL_DATABASE_SIZE_NOTSET;
try {
let prefs = Cc["@mozilla.org/preferences-service;1"].
getService(Ci.nsIPrefBranch);
cache_percentage =
prefs.getIntPref(PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE);
if (cache_percentage < 0) {
cache_percentage = 0;
}
else if (cache_percentage > 50) {
cache_percentage = 50;
}
}
catch(e) {}
let cachesize = memsize * cache_percentage / 100;
this._urisLimit = Math.max(MIN_URIS,
parseInt(cachesize / AVG_SIZE_PER_URIENTRY));
optimalDatabaseSize = this._prefBranch.getIntPref(PREF_OPTIMAL_DATABASE_SIZE);
} catch (ex) {}
this._urisLimit = Math.ceil(optimalDatabaseSize / URIENTRY_AVG_SIZE);
}
// Expose the calculated limit to other components.
this._prefBranch.setIntPref(PREF_READONLY_CALCULATED_MAX_URIS,
@ -827,6 +802,11 @@ nsPlacesExpiration.prototype = {
// Skip expiration during batch mode.
if (this._inBatchMode)
return;
// Don't try to further expire after shutdown.
if (this._shuttingDown &&
aAction != ACTION.SHUTDOWN && aAction != ACTION.CLEAN_SHUTDOWN) {
return;
}
let boundStatements = [];
for (let queryType in EXPIRATION_QUERIES) {
@ -901,18 +881,6 @@ nsPlacesExpiration.prototype = {
aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit;
break;
case "QUERY_FIND_URIS_TO_EXPIRE":
// We could run in the middle of adding a new visit or bookmark to
// a new page. In such a case since we are async, we could end up
// expiring the page before it actually gets the visit or bookmark,
// thinking it's an orphan. So we never expire the last added page
// when expiration does not run on user action.
if (aAction != ACTION.TIMED && aAction != ACTION.TIMED_OVERLIMIT &&
aAction != ACTION.IDLE) {
params.null_skips_last = -1;
}
else {
params.null_skips_last = null;
}
params.limit_uris = baseLimit;
break;
case "QUERY_SILENT_EXPIRE_ORPHAN_URIS":

View File

@ -128,3 +128,20 @@ function clearHistoryEnabled() {
}
catch(ex) {}
}
/**
* Returns a PRTime in the past usable to add expirable visits.
*
* @note Expiration ignores any visit added in the last 7 days, but it's
* better be safe against DST issues, by going back one day more.
*/
function getExpirablePRTime() {
let dateObj = new Date();
// Normalize to midnight
dateObj.setHours(0);
dateObj.setMinutes(0);
dateObj.setSeconds(0);
dateObj.setMilliseconds(0);
dateObj = new Date(dateObj.getTime() - 8 * 86400000);
return dateObj.getTime() * 1000;
}

View File

@ -62,7 +62,7 @@ function run_test() {
setMaxPages(0);
// Add some visited page and a couple expire with history annotations for each.
let now = Date.now() * 1000;
let now = getExpirablePRTime();
for (let i = 0; i < 5; i++) {
let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
hs.addVisit(pageURI, now++, null, hs.TRANSITION_TYPED, false, 0);

View File

@ -65,7 +65,7 @@ function run_test() {
setMaxPages(0);
// Add some visited page and a couple expire never annotations for each.
let now = Date.now() * 1000;
let now = getExpirablePRTime();
for (let i = 0; i < 5; i++) {
let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
hs.addVisit(pageURI, now++, null, hs.TRANSITION_TYPED, false, 0);

View File

@ -8,7 +8,7 @@
* only expire orphan entries, unless -1 is passed as limit.
*/
let gNow = Date.now() * 1000;
let gNow = getExpirablePRTime();
add_test(function test_expire_orphans()
{

View File

@ -100,7 +100,7 @@ function run_next_test() {
gCurrentTest.receivedNotifications = 0;
// Setup visits.
let now = Date.now() * 1000;
let now = getExpirablePRTime();
for (let i = 0; i < gCurrentTest.addPages; i++) {
let page = "http://" + gTestIndex + "." + i + ".mozilla.org/";
hs.addVisit(uri(page), now++, null, hs.TRANSITION_TYPED, false, 0);

View File

@ -117,7 +117,7 @@ function run_next_test() {
gCurrentTest.receivedNotifications = 0;
// Setup visits.
let now = Date.now() * 1000;
let now = getExpirablePRTime();
for (let j = 0; j < gCurrentTest.visitsPerPage; j++) {
for (let i = 0; i < gCurrentTest.addPages; i++) {
let page = "http://" + gTestIndex + "." + i + ".mozilla.org/";

View File

@ -121,7 +121,7 @@ function run_next_test() {
gCurrentTest.receivedNotifications = 0;
// Setup visits.
let now = Date.now() * 1000;
let now = getExpirablePRTime();
for (let i = 0; i < gCurrentTest.addPages; i++) {
hs.addVisit(uri("http://" + gTestIndex + "." + i + ".mozilla.org/"), now++, null,
hs.TRANSITION_TYPED, false, 0);

View File

@ -90,7 +90,7 @@ let historyObserver = {
stmt.finalize();
stmt = mDBConn.createStatement(
"SELECT h.id FROM moz_places h WHERE h.frecency = -2 " +
"SELECT h.id FROM moz_places h WHERE h.frecency < 0 " +
"AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1");
do_check_true(stmt.executeStep());
stmt.finalize();

View File

@ -382,8 +382,8 @@ var gTests = [
do_check_true(bmsvc.isBookmarked(TEST_URI));
waitForAsyncUpdates(function () {
print("Frecency should be -visit_count.")
do_check_eq(frecencyForUrl(TEST_URI), -10);
print("Frecency should be negative.")
do_check_true(frecencyForUrl(TEST_URI) < 0);
run_next_test();
});
}