Bug 853389 - Convert AddonRepository from SQLite to JSON. r=bmcbride

This commit is contained in:
Felipe Gomes 2013-08-01 12:12:40 -03:00
parent bbac896aaa
commit fba1d4eb50
6 changed files with 912 additions and 840 deletions

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,521 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const Cc = Components.classes;
const Ci = Components.interfaces;
Components.utils.import("resource://gre/modules/Services.jsm");
Components.utils.import("resource://gre/modules/AddonManager.jsm");
Components.utils.import("resource://gre/modules/FileUtils.jsm");
const KEY_PROFILEDIR = "ProfD";
const FILE_DATABASE = "addons.sqlite";
const LAST_DB_SCHEMA = 4;
// Add-on properties present in the columns of the database
const PROP_SINGLE = ["id", "type", "name", "version", "creator", "description",
"fullDescription", "developerComments", "eula",
"homepageURL", "supportURL", "contributionURL",
"contributionAmount", "averageRating", "reviewCount",
"reviewURL", "totalDownloads", "weeklyDownloads",
"dailyUsers", "sourceURI", "repositoryStatus", "size",
"updateDate"];
["LOG", "WARN", "ERROR"].forEach(function(aName) {
this.__defineGetter__(aName, function logFuncGetter() {
Components.utils.import("resource://gre/modules/AddonLogging.jsm");
LogManager.getLogger("addons.repository.sqlmigrator", this);
return this[aName];
});
}, this);
this.EXPORTED_SYMBOLS = ["AddonRepository_SQLiteMigrator"];
this.AddonRepository_SQLiteMigrator = {
/**
* Migrates data from a previous SQLite version of the
* database to the JSON version.
*
* @param structFunctions an object that contains functions
* to create the various objects used
* in the new JSON format
* @param aCallback A callback to be called when migration
* finishes, with the results in an array
* @returns bool True if a migration will happen (DB was
* found and succesfully opened)
*/
migrate: function(aCallback) {
if (!this._openConnection()) {
this._closeConnection();
aCallback([]);
return false;
}
LOG("Importing addon repository from previous " + FILE_DATABASE + " storage.");
this._retrieveStoredData((results) => {
this._closeConnection();
let resultArray = [addon for ([,addon] of Iterator(results))];
LOG(resultArray.length + " addons imported.")
aCallback(resultArray);
});
return true;
},
/**
* Synchronously opens a new connection to the database file.
*
* @return bool Whether the DB was opened successfully.
*/
_openConnection: function AD_openConnection() {
delete this.connection;
let dbfile = FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true);
if (!dbfile.exists())
return false;
try {
this.connection = Services.storage.openUnsharedDatabase(dbfile);
} catch (e) {
return false;
}
this.connection.executeSimpleSQL("PRAGMA locking_mode = EXCLUSIVE");
// Any errors in here should rollback
try {
this.connection.beginTransaction();
switch (this.connection.schemaVersion) {
case 0:
return false;
case 1:
LOG("Upgrading database schema to version 2");
this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN width INTEGER");
this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN height INTEGER");
this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN thumbnailWidth INTEGER");
this.connection.executeSimpleSQL("ALTER TABLE screenshot ADD COLUMN thumbnailHeight INTEGER");
case 2:
LOG("Upgrading database schema to version 3");
this.connection.createTable("compatibility_override",
"addon_internal_id INTEGER, " +
"num INTEGER, " +
"type TEXT, " +
"minVersion TEXT, " +
"maxVersion TEXT, " +
"appID TEXT, " +
"appMinVersion TEXT, " +
"appMaxVersion TEXT, " +
"PRIMARY KEY (addon_internal_id, num)");
case 3:
LOG("Upgrading database schema to version 4");
this.connection.createTable("icon",
"addon_internal_id INTEGER, " +
"size INTEGER, " +
"url TEXT, " +
"PRIMARY KEY (addon_internal_id, size)");
this._createIndices();
this._createTriggers();
this.connection.schemaVersion = LAST_DB_SCHEMA;
case LAST_DB_SCHEMA:
break;
default:
return false;
}
this.connection.commitTransaction();
} catch (e) {
ERROR("Failed to open " + FILE_DATABASE + ". Data import will not happen.", e);
this.logSQLError(this.connection.lastError, this.connection.lastErrorString);
this.connection.rollbackTransaction();
return false;
}
return true;
},
_closeConnection: function() {
for each (let stmt in this.asyncStatementsCache)
stmt.finalize();
this.asyncStatementsCache = {};
if (this.connection)
this.connection.asyncClose();
delete this.connection;
},
/**
* Asynchronously retrieve all add-ons from the database, and pass it
* to the specified callback
*
* @param aCallback
* The callback to pass the add-ons back to
*/
_retrieveStoredData: function AD_retrieveStoredData(aCallback) {
let self = this;
let addons = {};
// Retrieve all data from the addon table
function getAllAddons() {
self.getAsyncStatement("getAllAddons").executeAsync({
handleResult: function getAllAddons_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
let internal_id = row.getResultByName("internal_id");
addons[internal_id] = self._makeAddonFromAsyncRow(row);
}
},
handleError: self.asyncErrorLogger,
handleCompletion: function getAllAddons_handleCompletion(aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
ERROR("Error retrieving add-ons from database. Returning empty results");
aCallback({});
return;
}
getAllDevelopers();
}
});
}
// Retrieve all data from the developer table
function getAllDevelopers() {
self.getAsyncStatement("getAllDevelopers").executeAsync({
handleResult: function getAllDevelopers_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
let addon_internal_id = row.getResultByName("addon_internal_id");
if (!(addon_internal_id in addons)) {
WARN("Found a developer not linked to an add-on in database");
continue;
}
let addon = addons[addon_internal_id];
if (!addon.developers)
addon.developers = [];
addon.developers.push(self._makeDeveloperFromAsyncRow(row));
}
},
handleError: self.asyncErrorLogger,
handleCompletion: function getAllDevelopers_handleCompletion(aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
ERROR("Error retrieving developers from database. Returning empty results");
aCallback({});
return;
}
getAllScreenshots();
}
});
}
// Retrieve all data from the screenshot table
function getAllScreenshots() {
self.getAsyncStatement("getAllScreenshots").executeAsync({
handleResult: function getAllScreenshots_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
let addon_internal_id = row.getResultByName("addon_internal_id");
if (!(addon_internal_id in addons)) {
WARN("Found a screenshot not linked to an add-on in database");
continue;
}
let addon = addons[addon_internal_id];
if (!addon.screenshots)
addon.screenshots = [];
addon.screenshots.push(self._makeScreenshotFromAsyncRow(row));
}
},
handleError: self.asyncErrorLogger,
handleCompletion: function getAllScreenshots_handleCompletion(aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
ERROR("Error retrieving screenshots from database. Returning empty results");
aCallback({});
return;
}
getAllCompatOverrides();
}
});
}
function getAllCompatOverrides() {
self.getAsyncStatement("getAllCompatOverrides").executeAsync({
handleResult: function getAllCompatOverrides_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
let addon_internal_id = row.getResultByName("addon_internal_id");
if (!(addon_internal_id in addons)) {
WARN("Found a compatibility override not linked to an add-on in database");
continue;
}
let addon = addons[addon_internal_id];
if (!addon.compatibilityOverrides)
addon.compatibilityOverrides = [];
addon.compatibilityOverrides.push(self._makeCompatOverrideFromAsyncRow(row));
}
},
handleError: self.asyncErrorLogger,
handleCompletion: function getAllCompatOverrides_handleCompletion(aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
ERROR("Error retrieving compatibility overrides from database. Returning empty results");
aCallback({});
return;
}
getAllIcons();
}
});
}
function getAllIcons() {
self.getAsyncStatement("getAllIcons").executeAsync({
handleResult: function getAllIcons_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
let addon_internal_id = row.getResultByName("addon_internal_id");
if (!(addon_internal_id in addons)) {
WARN("Found an icon not linked to an add-on in database");
continue;
}
let addon = addons[addon_internal_id];
let { size, url } = self._makeIconFromAsyncRow(row);
addon.icons[size] = url;
if (size == 32)
addon.iconURL = url;
}
},
handleError: self.asyncErrorLogger,
handleCompletion: function getAllIcons_handleCompletion(aReason) {
if (aReason != Ci.mozIStorageStatementCallback.REASON_FINISHED) {
ERROR("Error retrieving icons from database. Returning empty results");
aCallback({});
return;
}
let returnedAddons = {};
for each (let addon in addons)
returnedAddons[addon.id] = addon;
aCallback(returnedAddons);
}
});
}
// Begin asynchronous process
getAllAddons();
},
// A cache of statements that are used and need to be finalized on shutdown
asyncStatementsCache: {},
/**
* Gets a cached async statement or creates a new statement if it doesn't
* already exist.
*
* @param aKey
* A unique key to reference the statement
* @return a mozIStorageAsyncStatement for the SQL corresponding to the
* unique key
*/
getAsyncStatement: function AD_getAsyncStatement(aKey) {
if (aKey in this.asyncStatementsCache)
return this.asyncStatementsCache[aKey];
let sql = this.queries[aKey];
try {
return this.asyncStatementsCache[aKey] = this.connection.createAsyncStatement(sql);
} catch (e) {
ERROR("Error creating statement " + aKey + " (" + sql + ")");
throw Components.Exception("Error creating statement " + aKey + " (" + sql + "): " + e,
e.result);
}
},
// The queries used by the database
queries: {
getAllAddons: "SELECT internal_id, id, type, name, version, " +
"creator, creatorURL, description, fullDescription, " +
"developerComments, eula, homepageURL, supportURL, " +
"contributionURL, contributionAmount, averageRating, " +
"reviewCount, reviewURL, totalDownloads, weeklyDownloads, " +
"dailyUsers, sourceURI, repositoryStatus, size, updateDate " +
"FROM addon",
getAllDevelopers: "SELECT addon_internal_id, name, url FROM developer " +
"ORDER BY addon_internal_id, num",
getAllScreenshots: "SELECT addon_internal_id, url, width, height, " +
"thumbnailURL, thumbnailWidth, thumbnailHeight, caption " +
"FROM screenshot ORDER BY addon_internal_id, num",
getAllCompatOverrides: "SELECT addon_internal_id, type, minVersion, " +
"maxVersion, appID, appMinVersion, appMaxVersion " +
"FROM compatibility_override " +
"ORDER BY addon_internal_id, num",
getAllIcons: "SELECT addon_internal_id, size, url FROM icon " +
"ORDER BY addon_internal_id, size",
},
/**
* Make add-on structure from an asynchronous row.
*
* @param aRow
* The asynchronous row to use
* @return The created add-on
*/
_makeAddonFromAsyncRow: function AD__makeAddonFromAsyncRow(aRow) {
// This is intentionally not an AddonSearchResult object in order
// to allow AddonDatabase._parseAddon to parse it, same as if it
// was read from the JSON database.
let addon = { icons: {} };
for (let prop of PROP_SINGLE) {
addon[prop] = aRow.getResultByName(prop)
};
return addon;
},
/**
* Make a developer from an asynchronous row
*
* @param aRow
* The asynchronous row to use
* @return The created developer
*/
_makeDeveloperFromAsyncRow: function AD__makeDeveloperFromAsyncRow(aRow) {
let name = aRow.getResultByName("name");
let url = aRow.getResultByName("url")
return new AddonManagerPrivate.AddonAuthor(name, url);
},
/**
* Make a screenshot from an asynchronous row
*
* @param aRow
* The asynchronous row to use
* @return The created screenshot
*/
_makeScreenshotFromAsyncRow: function AD__makeScreenshotFromAsyncRow(aRow) {
let url = aRow.getResultByName("url");
let width = aRow.getResultByName("width");
let height = aRow.getResultByName("height");
let thumbnailURL = aRow.getResultByName("thumbnailURL");
let thumbnailWidth = aRow.getResultByName("thumbnailWidth");
let thumbnailHeight = aRow.getResultByName("thumbnailHeight");
let caption = aRow.getResultByName("caption");
return new AddonManagerPrivate.AddonScreenshot(url, width, height, thumbnailURL,
thumbnailWidth, thumbnailHeight, caption);
},
/**
* Make a CompatibilityOverride from an asynchronous row
*
* @param aRow
* The asynchronous row to use
* @return The created CompatibilityOverride
*/
_makeCompatOverrideFromAsyncRow: function AD_makeCompatOverrideFromAsyncRow(aRow) {
let type = aRow.getResultByName("type");
let minVersion = aRow.getResultByName("minVersion");
let maxVersion = aRow.getResultByName("maxVersion");
let appID = aRow.getResultByName("appID");
let appMinVersion = aRow.getResultByName("appMinVersion");
let appMaxVersion = aRow.getResultByName("appMaxVersion");
return new AddonManagerPrivate.AddonCompatibilityOverride(type,
minVersion,
maxVersion,
appID,
appMinVersion,
appMaxVersion);
},
/**
* Make an icon from an asynchronous row
*
* @param aRow
* The asynchronous row to use
* @return An object containing the size and URL of the icon
*/
_makeIconFromAsyncRow: function AD_makeIconFromAsyncRow(aRow) {
let size = aRow.getResultByName("size");
let url = aRow.getResultByName("url");
return { size: size, url: url };
},
/**
* A helper function to log an SQL error.
*
* @param aError
* The storage error code associated with the error
* @param aErrorString
* An error message
*/
logSQLError: function AD_logSQLError(aError, aErrorString) {
ERROR("SQL error " + aError + ": " + aErrorString);
},
/**
* A helper function to log any errors that occur during async statements.
*
* @param aError
* A mozIStorageError to log
*/
asyncErrorLogger: function AD_asyncErrorLogger(aError) {
ERROR("Async SQL error " + aError.result + ": " + aError.message);
},
/**
* Synchronously creates the triggers in the database.
*/
_createTriggers: function AD__createTriggers() {
this.connection.executeSimpleSQL("DROP TRIGGER IF EXISTS delete_addon");
this.connection.executeSimpleSQL("CREATE TRIGGER delete_addon AFTER DELETE " +
"ON addon BEGIN " +
"DELETE FROM developer WHERE addon_internal_id=old.internal_id; " +
"DELETE FROM screenshot WHERE addon_internal_id=old.internal_id; " +
"DELETE FROM compatibility_override WHERE addon_internal_id=old.internal_id; " +
"DELETE FROM icon WHERE addon_internal_id=old.internal_id; " +
"END");
},
/**
* Synchronously creates the indices in the database.
*/
_createIndices: function AD__createIndices() {
this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS developer_idx " +
"ON developer (addon_internal_id)");
this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS screenshot_idx " +
"ON screenshot (addon_internal_id)");
this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS compatibility_override_idx " +
"ON compatibility_override (addon_internal_id)");
this.connection.executeSimpleSQL("CREATE INDEX IF NOT EXISTS icon_idx " +
"ON icon (addon_internal_id)");
}
}

View File

@ -29,6 +29,7 @@ EXTRA_PP_COMPONENTS += [
EXTRA_JS_MODULES += [
'AddonLogging.jsm',
'AddonRepository.jsm',
'AddonRepository_SQLiteMigrator.jsm',
'AddonUpdateChecker.jsm',
'ChromeManifestParser.jsm',
'DeferredSave.jsm',

View File

@ -615,7 +615,7 @@ function createInstallRDF(aData) {
function writeInstallRDFToDir(aData, aDir, aExtraFile) {
var rdf = createInstallRDF(aData);
if (!aDir.exists())
aDir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755);
aDir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
var file = aDir.clone();
file.append("install.rdf");
if (file.exists())
@ -633,7 +633,7 @@ function writeInstallRDFToDir(aData, aDir, aExtraFile) {
file = aDir.clone();
file.append(aExtraFile);
file.create(AM_Ci.nsIFile.NORMAL_FILE_TYPE, 0644);
file.create(AM_Ci.nsIFile.NORMAL_FILE_TYPE, FileUtils.PERMS_FILE);
}
/**
@ -665,7 +665,7 @@ function writeInstallRDFForExtension(aData, aDir, aId, aExtraFile) {
}
if (!dir.exists())
dir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755);
dir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
dir.append(id + ".xpi");
var rdf = createInstallRDF(aData);
var stream = AM_Cc["@mozilla.org/io/string-input-stream;1"].
@ -718,7 +718,7 @@ function manuallyInstall(aXPIFile, aInstallLocation, aID) {
if (TEST_UNPACKED) {
let dir = aInstallLocation.clone();
dir.append(aID);
dir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755);
dir.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
let zip = AM_Cc["@mozilla.org/libjar/zip-reader;1"].
createInstance(AM_Ci.nsIZipReader);
zip.open(aXPIFile);

View File

@ -20,7 +20,7 @@ const GETADDONS_RESULTS = BASE_URL + "/data/test_AddonRepository_cach
const GETADDONS_EMPTY = BASE_URL + "/data/test_AddonRepository_empty.xml";
const GETADDONS_FAILED = BASE_URL + "/data/test_AddonRepository_failed.xml";
const FILE_DATABASE = "addons.sqlite";
const FILE_DATABASE = "addons.json";
const ADDON_NAMES = ["test_AddonRepository_1",
"test_AddonRepository_2",
"test_AddonRepository_3"];
@ -521,6 +521,16 @@ function check_initialized_cache(aExpectedToFind, aCallback) {
});
}
// Waits for the data to be written from the in-memory DB to the addons.json
// file that is done asynchronously through OS.File
function waitForFlushedData(aCallback) {
Services.obs.addObserver({
observe: function(aSubject, aTopic, aData) {
Services.obs.removeObserver(this, "addon-repository-data-written");
aCallback(aData == "true");
}
}, "addon-repository-data-written", false);
}
function run_test() {
// Setup for test
@ -558,7 +568,8 @@ function run_test_1() {
// Tests that the cache and database begin as empty
function run_test_2() {
check_database_exists(false);
check_cache([false, false, false], false, run_test_3);
check_cache([false, false, false], false, function(){});
waitForFlushedData(run_test_3);
}
// Tests repopulateCache when the search fails
@ -611,7 +622,9 @@ function run_test_6() {
check_database_exists(false);
Services.prefs.setBoolPref(PREF_GETADDONS_CACHE_ENABLED, true);
check_cache([false, false, false], false, run_test_7);
check_cache([false, false, false], false, function() {});
waitForFlushedData(run_test_7);
});
}
@ -710,7 +723,7 @@ function run_test_13() {
function run_test_14() {
Services.prefs.setBoolPref(PREF_GETADDONS_CACHE_ENABLED, true);
trigger_background_update(function() {
waitForFlushedData(function() {
check_database_exists(true);
AddonManager.getAddonsByIDs(ADDON_IDS, function(aAddons) {
@ -718,6 +731,8 @@ function run_test_14() {
do_execute_soon(run_test_15);
});
});
trigger_background_update();
}
// Tests that the XPI add-ons correctly use the repository properties when

View File

@ -109,7 +109,7 @@ function run_test() {
stmt.finalize();
db.close();
run_test_2();
do_test_finished();
}
}, "addon-repository-shutdown", null);
@ -126,37 +126,3 @@ function run_test() {
AddonRepository.shutdown();
});
}
function run_test_2() {
// Write out a minimal database.
let db = AM_Cc["@mozilla.org/storage/service;1"].
getService(AM_Ci.mozIStorageService).
openDatabase(dbfile);
db.createTable("futuristicSchema",
"id INTEGER, " +
"sharks TEXT, " +
"lasers TEXT");
db.schemaVersion = 1000;
db.close();
Services.obs.addObserver({
observe: function () {
Services.obs.removeObserver(this, "addon-repository-shutdown");
// Check the DB schema has changed once AddonRepository has freed it.
db = AM_Cc["@mozilla.org/storage/service;1"].
getService(AM_Ci.mozIStorageService).
openDatabase(dbfile);
do_check_eq(db.schemaVersion, EXPECTED_SCHEMA_VERSION);
db.close();
do_test_finished();
}
}, "addon-repository-shutdown", null);
// Force a connection to the addon database to be opened.
Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", true);
AddonRepository.getCachedAddonByID("test1@tests.mozilla.org", function (aAddon) {
AddonRepository.shutdown();
});
}