wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 853388: Upgrade existing SQLITE databases to JSON; r=unfocused

Browse Source
This commit is contained in:
Irving Reid 2013-08-08 15:56:26 -04:00
parent 0033499ba4
commit 38feab6f09
12 changed files with 320 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
toolkit/mozapps/extensions/XPIProvider.jsm
View File

@ -595,14 +595,8 @@ function isAddonDisabled(aAddon) {
return aAddon.appDisabled || aAddon.softDisabled || aAddon.userDisabled;
}
Object.defineProperty(this, "gRDF", {
get: function gRDFGetter() {
delete this.gRDF;
return this.gRDF = Cc["@mozilla.org/rdf/rdf-service;1"].
getService(Ci.nsIRDFService);
},
configurable: true
});
XPCOMUtils.defineLazyServiceGetter(this, "gRDF", "@mozilla.org/rdf/rdf-service;1",
Ci.nsIRDFService);
function EM_R(aProperty) {
return gRDF.GetResource(PREFIX_NS_EM + aProperty);
@ -1765,7 +1759,7 @@ var XPIProvider = {
null);
this.minCompatiblePlatformVersion = Prefs.getCharPref(PREF_EM_MIN_COMPAT_PLATFORM_VERSION,
null);
this.enabledAddons = [];
this.enabledAddons = "";
Services.prefs.addObserver(PREF_EM_MIN_COMPAT_APP_VERSION, this, false);
Services.prefs.addObserver(PREF_EM_MIN_COMPAT_PLATFORM_VERSION, this, false);
@ -2904,6 +2898,7 @@ var XPIProvider = {
let newDBAddon = null;
try {
// Update the database.
// XXX I don't think this can throw any more
newDBAddon = XPIDatabase.addAddonMetadata(newAddon, aAddonState.descriptor);
}
catch (e) {
@ -3005,9 +3000,8 @@ var XPIProvider = {
let addonStates = aSt.addons;
// Check if the database knows about any add-ons in this install location.
let pos = knownLocations.indexOf(installLocation.name);
if (pos >= 0) {
knownLocations.splice(pos, 1);
if (knownLocations.has(installLocation.name)) {
knownLocations.delete(installLocation.name);
let addons = XPIDatabase.getAddonsInLocation(installLocation.name);
// Iterate through the add-ons installed the last time the application
// ran
@ -3084,12 +3078,12 @@ var XPIProvider = {
// have any add-ons installed in them, or the locations no longer exist.
// The metadata for the add-ons that were in them must be removed from the
// database.
knownLocations.forEach(function(aLocation) {
let addons = XPIDatabase.getAddonsInLocation(aLocation);
for (let location of knownLocations) {
let addons = XPIDatabase.getAddonsInLocation(location);
addons.forEach(function(aOldAddon) {
changed = removeMetadata(aOldAddon) || changed;
}, this);
}, this);
}
// Tell Telemetry what we found
AddonManagerPrivate.recordSimpleMeasure("modifiedUnpacked", modifiedUnpacked);
@ -5379,6 +5373,8 @@ AddonInstall.prototype = {
reason, extraParams);
}
else {
// XXX this makes it dangerous to do many things in onInstallEnded
// listeners because important cleanup hasn't been done yet
XPIProvider.unloadBootstrapScope(this.addon.id);
}
}
@ -5756,8 +5752,8 @@ UpdateChecker.prototype = {
/**
* The AddonInternal is an internal only representation of add-ons. It may
* have come from the database (see DBAddonInternal below) or an install
* manifest.
* have come from the database (see DBAddonInternal in XPIProviderUtils.jsm)
* or an install manifest.
*/
function AddonInternal() {
}

609
toolkit/mozapps/extensions/XPIProviderUtils.js
View File

@ -16,7 +16,6 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
"resource://gre/modules/FileUtils.jsm");
["LOG", "WARN", "ERROR"].forEach(function(aName) {
Object.defineProperty(this, aName, {
get: function logFuncGetter () {
@ -93,14 +92,8 @@ const PREFIX_ITEM_URI = "urn:mozilla:item:";
const RDFURI_ITEM_ROOT = "urn:mozilla:item:root"
const PREFIX_NS_EM = "http://www.mozilla.org/2004/em-rdf#";
Object.defineProperty(this, "gRDF", {
get: function gRDFGetter() {
delete this.gRDF;
return this.gRDF = Cc["@mozilla.org/rdf/rdf-service;1"].
getService(Ci.nsIRDFService);
},
configurable: true
});
XPCOMUtils.defineLazyServiceGetter(this, "gRDF", "@mozilla.org/rdf/rdf-service;1",
Ci.nsIRDFService);
function EM_R(aProperty) {
return gRDF.GetResource(PREFIX_NS_EM + aProperty);
@ -138,60 +131,6 @@ function getRDFProperty(aDs, aResource, aProperty) {
return getRDFValue(aDs.GetTarget(aResource, EM_R(aProperty), true));
}
/**
* A mozIStorageStatementCallback that will asynchronously build DBAddonInternal
* instances from the results it receives. Once the statement has completed
* executing and all of the metadata for all of the add-ons has been retrieved
* they will be passed as an array to aCallback.
*
* @param aCallback
* A callback function to pass the array of DBAddonInternals to
*/
function AsyncAddonListCallback(aCallback) {
this.callback = aCallback;
this.addons = [];
}
AsyncAddonListCallback.prototype = {
callback: null,
complete: false,
count: 0,
addons: null,
handleResult: function AsyncAddonListCallback_handleResult(aResults) {
let row = null;
while ((row = aResults.getNextRow())) {
this.count++;
let self = this;
XPIDatabase.makeAddonFromRowAsync(row, function handleResult_makeAddonFromRowAsync(aAddon) {
function completeAddon(aRepositoryAddon) {
aAddon._repositoryAddon = aRepositoryAddon;
aAddon.compatibilityOverrides = aRepositoryAddon ?
aRepositoryAddon.compatibilityOverrides :
null;
self.addons.push(aAddon);
if (self.complete && self.addons.length == self.count)
self.callback(self.addons);
}
if ("getCachedAddonByID" in AddonRepository)
AddonRepository.getCachedAddonByID(aAddon.id, completeAddon);
else
completeAddon(null);
});
}
},
handleError: asyncErrorLogger,
handleCompletion: function AsyncAddonListCallback_handleCompletion(aReason) {
this.complete = true;
if (this.addons.length == this.count)
this.callback(this.addons);
}
};
/**
* Asynchronously fill in the _repositoryAddon field for one addon
*/
@ -293,24 +232,6 @@ function asyncErrorLogger(aError) {
logSQLError(aError.result, aError.message);
}
/**
* A helper function to execute a statement synchronously and log any error
* that occurs.
*
* @param aStatement
* A mozIStorageStatement to execute
*/
function executeStatement(aStatement) {
try {
aStatement.execute();
}
catch (e) {
logSQLError(XPIDatabase.connection.lastError,
XPIDatabase.connection.lastErrorString);
throw e;
}
}
/**
* A helper function to step a statement synchronously and log any error that
* occurs.
@ -372,12 +293,6 @@ function copyRowProperties(aRow, aProperties, aTarget) {
return aTarget;
}
/**
* Create a DBAddonInternal from the fields saved in the JSON database
* or loaded into an AddonInternal from an XPI manifest.
* @return a DBAddonInternal populated with the loaded data
*/
/**
* The DBAddonInternal is a special AddonInternal that has been retrieved from
* the database. The constructor will initialize the DBAddonInternal with a set
@ -389,6 +304,7 @@ function copyRowProperties(aRow, aProperties, aTarget) {
*/
function DBAddonInternal(aLoaded) {
copyProperties(aLoaded, PROP_JSON_FIELDS, this);
if (aLoaded._installLocation) {
this._installLocation = aLoaded._installLocation;
this.location = aLoaded._installLocation._name;
@ -396,7 +312,9 @@ function DBAddonInternal(aLoaded) {
else if (aLoaded.location) {
this._installLocation = XPIProvider.installLocationsByName[this.location];
}
this._key = this.location + ":" + this.id;
try {
this._sourceBundle = this._installLocation.getLocationForID(this.id);
}
@ -406,20 +324,20 @@ function DBAddonInternal(aLoaded) {
// this change is being detected.
}
Object.defineProperty(this, "pendingUpgrade", {
get: function DBA_pendingUpgradeGetter() {
delete this.pendingUpgrade;
// XXX Can we redesign pendingUpgrade?
XPCOMUtils.defineLazyGetter(this, "pendingUpgrade",
function DBA_pendingUpgradeGetter() {
for (let install of XPIProvider.installs) {
if (install.state == AddonManager.STATE_INSTALLED &&
!(install.addon.inDatabase) &&
install.addon.id == this.id &&
install.installLocation == this._installLocation) {
delete this.pendingUpgrade;
return this.pendingUpgrade = install.addon;
}
};
},
configurable: true
});
return null;
});
}
DBAddonInternal.prototype = {
@ -437,8 +355,13 @@ DBAddonInternal.prototype = {
XPIProvider.updateAddonDisabledState(this);
XPIDatabase.commitTransaction();
},
get inDatabase() {
return true;
},
toJSON: function() {
return copyProperties(this, PROP_JSON_FIELDS);
}
}
@ -447,94 +370,21 @@ DBAddonInternal.prototype.__proto__ = AddonInternal.prototype;
this.XPIDatabase = {
// true if the database connection has been opened
initialized: false,
// A cache of statements that are used and need to be finalized on shutdown
statementCache: {},
// A cache of weak referenced DBAddonInternals so we can reuse objects where
// possible
addonCache: [],
// The nested transaction count
transactionCount: 0,
// The database file
dbfile: FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true),
jsonFile: FileUtils.getFile(KEY_PROFILEDIR, [FILE_JSON_DB], true),
// Migration data loaded from an old version of the database.
migrateData: null,
// Active add-on directories loaded from extensions.ini and prefs at startup.
activeBundles: null,
// Special handling for when the database is locked at first load
lockedDatabase: false,
// The statements used by the database
statements: {
_getDefaultLocale: "SELECT id, name, description, creator, homepageURL " +
"FROM locale WHERE id=:id",
_getLocales: "SELECT addon_locale.locale, locale.id, locale.name, " +
"locale.description, locale.creator, locale.homepageURL " +
"FROM addon_locale JOIN locale ON " +
"addon_locale.locale_id=locale.id WHERE " +
"addon_internal_id=:internal_id",
_getTargetApplications: "SELECT addon_internal_id, id, minVersion, " +
"maxVersion FROM targetApplication WHERE " +
"addon_internal_id=:internal_id",
_getTargetPlatforms: "SELECT os, abi FROM targetPlatform WHERE " +
"addon_internal_id=:internal_id",
_readLocaleStrings: "SELECT locale_id, type, value FROM locale_strings " +
"WHERE locale_id=:id",
clearVisibleAddons: "UPDATE addon SET visible=0 WHERE id=:id",
updateAddonActive: "UPDATE addon SET active=:active WHERE " +
"internal_id=:internal_id",
getActiveAddons: "SELECT " + FIELDS_ADDON + " FROM addon WHERE active=1 AND " +
"type<>'theme' AND bootstrap=0",
getActiveTheme: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " +
"internalName=:internalName AND type='theme'",
getThemes: "SELECT " + FIELDS_ADDON + " FROM addon WHERE type='theme'",
getAddonInLocation: "SELECT " + FIELDS_ADDON + " FROM addon WHERE id=:id " +
"AND location=:location",
getAddons: "SELECT " + FIELDS_ADDON + " FROM addon",
getAddonsByType: "SELECT " + FIELDS_ADDON + " FROM addon WHERE type=:type",
getAddonsInLocation: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " +
"location=:location",
getInstallLocations: "SELECT DISTINCT location FROM addon",
getVisibleAddonForID: "SELECT " + FIELDS_ADDON + " FROM addon WHERE " +
"visible=1 AND id=:id",
getVisibleAddonForInternalName: "SELECT " + FIELDS_ADDON + " FROM addon " +
"WHERE visible=1 AND internalName=:internalName",
getVisibleAddons: "SELECT " + FIELDS_ADDON + " FROM addon WHERE visible=1",
getVisibleAddonsWithPendingOperations: "SELECT " + FIELDS_ADDON + " FROM " +
"addon WHERE visible=1 " +
"AND (pendingUninstall=1 OR " +
"MAX(userDisabled,appDisabled)=active)",
getAddonBySyncGUID: "SELECT " + FIELDS_ADDON + " FROM addon " +
"WHERE syncGUID=:syncGUID",
makeAddonVisible: "UPDATE addon SET visible=1 WHERE internal_id=:internal_id",
removeAddonMetadata: "DELETE FROM addon WHERE internal_id=:internal_id",
// Equates to active = visible && !userDisabled && !softDisabled &&
// !appDisabled && !pendingUninstall
setActiveAddons: "UPDATE addon SET active=MIN(visible, 1 - userDisabled, " +
"1 - softDisabled, 1 - appDisabled, 1 - pendingUninstall)",
setAddonProperties: "UPDATE addon SET userDisabled=:userDisabled, " +
"appDisabled=:appDisabled, " +
"softDisabled=:softDisabled, " +
"pendingUninstall=:pendingUninstall, " +
"applyBackgroundUpdates=:applyBackgroundUpdates WHERE " +
"internal_id=:internal_id",
setAddonDescriptor: "UPDATE addon SET descriptor=:descriptor WHERE " +
"internal_id=:internal_id",
setAddonSyncGUID: "UPDATE addon SET syncGUID=:syncGUID WHERE " +
"internal_id=:internal_id",
updateTargetApplications: "UPDATE targetApplication SET " +
"minVersion=:minVersion, maxVersion=:maxVersion " +
"WHERE addon_internal_id=:internal_id AND id=:id",
createSavepoint: "SAVEPOINT 'default'",
releaseSavepoint: "RELEASE SAVEPOINT 'default'",
rollbackSavepoint: "ROLLBACK TO SAVEPOINT 'default'"
},
// XXX may be able to refactor this away
get dbfileExists() {
delete this.dbfileExists;
return this.dbfileExists = this.dbfile.exists();
return this.dbfileExists = this.jsonFile.exists();
},
set dbfileExists(aValue) {
delete this.dbfileExists;
@ -545,12 +395,19 @@ this.XPIDatabase = {
* Converts the current internal state of the XPI addon database to JSON
* and writes it to the user's profile. Synchronous for now, eventually must
* be async, reliable, etc.
* XXX should we remove the JSON file if it would be empty? Not sure if that
* would ever happen, given the default theme
*/
writeJSON: function XPIDB_writeJSON() {
// XXX should have a guard here for if the addonDB hasn't been auto-loaded yet
// Don't mess with an existing database on disk, if it was locked at start up
if (this.lockedDatabase)
return;
let addons = [];
for (let aKey in this.addonDB) {
addons.push(copyProperties(this.addonDB[aKey], PROP_JSON_FIELDS));
for (let [key, addon] of this.addonDB) {
addons.push(addon);
}
let toSave = {
schemaVersion: DB_SCHEMA,
@ -563,11 +420,17 @@ this.XPIDatabase = {
try {
converter.init(stream, "UTF-8", 0, 0x0000);
// XXX pretty print the JSON while debugging
converter.writeString(JSON.stringify(toSave, null, 2));
let out = JSON.stringify(toSave, null, 2);
// dump("Writing JSON:\n" + out + "\n");
converter.writeString(out);
converter.flush();
// nsConverterOutputStream doesn't finish() safe output streams on close()
FileUtils.closeSafeFileOutputStream(stream);
converter.close();
this.dbfileExists = true;
// XXX probably only want to do this if the version is different
Services.prefs.setIntPref(PREF_DB_SCHEMA, DB_SCHEMA);
Services.prefs.savePrefFile(null); // XXX is this bad sync I/O?
}
catch(e) {
ERROR("Failed to save database to JSON", e);
@ -575,66 +438,6 @@ this.XPIDatabase = {
}
},
/**
* Open and parse the JSON XPI extensions database.
* @return true: the DB was successfully loaded
* false: The DB either needs upgrade or did not exist at all.
* XXX upgrade and errors handled in a following patch
*/
openJSONDatabase: function XPIDB_openJSONDatabase() {
dump("XPIDB_openJSONDatabase\n");
try {
let data = "";
let fstream = Components.classes["@mozilla.org/network/file-input-stream;1"].
createInstance(Components.interfaces.nsIFileInputStream);
let cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
createInstance(Components.interfaces.nsIConverterInputStream);
fstream.init(this.jsonFile, -1, 0, 0);
cstream.init(fstream, "UTF-8", 0, 0);
let (str = {}) {
let read = 0;
do {
read = cstream.readString(0xffffffff, str); // read as much as we can and put it in str.value
data += str.value;
} while (read != 0);
}
cstream.close();
let inputAddons = JSON.parse(data);
// Now do some sanity checks on our JSON db
if (!("schemaVersion" in inputAddons) || !("addons" in inputAddons)) {
// XXX Content of JSON file is bad, need to rebuild from scratch
ERROR("bad JSON file contents");
delete this.addonDB;
this.addonDB = {};
return false;
}
if (inputAddons.schemaVersion != DB_SCHEMA) {
// XXX UPGRADE FROM PREVIOUS VERSION OF JSON DB
ERROR("JSON schema upgrade needed");
return false;
}
// If we got here, we probably have good data
// Make AddonInternal instances from the loaded data and save them
delete this.addonDB;
let addonDB = {}
inputAddons.addons.forEach(function(loadedAddon) {
let newAddon = new DBAddonInternal(loadedAddon);
addonDB[newAddon._key] = newAddon;
});
this.addonDB = addonDB;
// dump("Finished reading DB: " + this.addonDB.toSource() + "\n");
return true;
}
catch(e) {
// XXX handle missing JSON database
ERROR("Failed to load XPI JSON data from profile", e);
// XXX for now, start from scratch
delete this.addonDB;
this.addonDB = {};
return false;
}
},
/**
* Begins a new transaction in the database. Transactions may be nested. Data
* written by an inner transaction may be rolled back on its own. Rolling back
@ -681,170 +484,197 @@ this.XPIDatabase = {
},
/**
* Attempts to open the database file. If it fails it will try to delete the
* existing file and create an empty database. If that fails then it will
* open an in-memory database that can be used during this session.
* Pull upgrade information from an existing SQLITE database
*
* @param aDBFile
* The nsIFile to open
* @return the mozIStorageConnection for the database
* @return false if there is no SQLITE database
* true and sets this.migrateData to null if the SQLITE DB exists
* but does not contain useful information
* true and sets this.migrateData to
* {location: {id1:{addon1}, id2:{addon2}}, location2:{...}, ...}
* if there is useful information
*/
openDatabaseFile: function XPIDB_openDatabaseFile(aDBFile) {
LOG("Opening database");
loadSqliteData: function XPIDB_loadSqliteData() {
let connection = null;
let dbfile = FileUtils.getFile(KEY_PROFILEDIR, [FILE_DATABASE], true);
if (!dbfile.exists()) {
return false;
}
// Attempt to open the database
try {
connection = Services.storage.openUnsharedDatabase(aDBFile);
this.dbfileExists = true;
connection = Services.storage.openUnsharedDatabase(dbfile);
}
catch (e) {
ERROR("Failed to open database (1st attempt)", e);
// If the database was locked for some reason then assume it still
// has some good data and we should try to load it the next time around.
if (e.result != Cr.NS_ERROR_STORAGE_BUSY) {
try {
aDBFile.remove(true);
}
catch (e) {
ERROR("Failed to remove database that could not be opened", e);
}
try {
connection = Services.storage.openUnsharedDatabase(aDBFile);
}
catch (e) {
ERROR("Failed to open database (2nd attempt)", e);
// If we have got here there seems to be no way to open the real
// database, instead open a temporary memory database so things will
// work for this session.
return Services.storage.openSpecialDatabase("memory");
}
}
else {
return Services.storage.openSpecialDatabase("memory");
}
// exists but SQLITE can't open it
WARN("Failed to open sqlite database " + dbfile.path + " for upgrade", e);
this.migrateData = null;
return true;
}
connection.executeSimpleSQL("PRAGMA synchronous = FULL");
connection.executeSimpleSQL("PRAGMA locking_mode = EXCLUSIVE");
return connection;
LOG("Migrating data from sqlite");
this.migrateData = this.getMigrateDataFromDatabase(connection);
connection.close();
return true;
},
/**
* Opens a new connection to the database file.
* Opens and reads the database file, upgrading from old
* databases or making a new DB if needed.
*
* The possibilities, in order of priority, are:
* 1) Perfectly good, up to date database
* 2) Out of date JSON database needs to be upgraded => upgrade
* 3) JSON database exists but is mangled somehow => build new JSON
* 4) no JSON DB, but a useable SQLITE db we can upgrade from => upgrade
* 5) useless SQLITE DB => build new JSON
* 6) useable RDF DB => upgrade
* 7) useless RDF DB => build new JSON
* 8) Nothing at all => build new JSON
* @param aRebuildOnError
* A boolean indicating whether add-on information should be loaded
* from the install locations if the database needs to be rebuilt.
* (if false, caller is XPIProvider.checkForChanges() which will rebuild)
*/
openConnection: function XPIDB_openConnection(aRebuildOnError, aForceOpen) {
this.openJSONDatabase();
// XXX TELEMETRY report opens with aRebuildOnError true (which implies delayed open)
// vs. aRebuildOnError false (DB loaded during startup)
delete this.addonDB;
this.migrateData = null;
let fstream = null;
let data = "";
try {
LOG("Opening XPI database " + this.jsonFile.path);
fstream = Components.classes["@mozilla.org/network/file-input-stream;1"].
createInstance(Components.interfaces.nsIFileInputStream);
fstream.init(this.jsonFile, -1, 0, 0);
let cstream = null;
try {
cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
createInstance(Components.interfaces.nsIConverterInputStream);
cstream.init(fstream, "UTF-8", 0, 0);
let (str = {}) {
let read = 0;
do {
read = cstream.readString(0xffffffff, str); // read as much as we can and put it in str.value
data += str.value;
} while (read != 0);
}
// dump("Loaded JSON:\n" + data + "\n");
let inputAddons = JSON.parse(data);
// Now do some sanity checks on our JSON db
if (!("schemaVersion" in inputAddons) || !("addons" in inputAddons)) {
// Content of JSON file is bad, need to rebuild from scratch
ERROR("bad JSON file contents");
this.rebuildDatabase(aRebuildOnError);
}
if (inputAddons.schemaVersion != DB_SCHEMA) {
// Handle mismatched JSON schema version. For now, we assume backward/forward
// compatibility as long as we preserve unknown fields during save & restore
// XXX preserve schema version and unknown fields during save/restore
LOG("JSON schema mismatch: expected " + DB_SCHEMA +
", actual " + inputAddons.schemaVersion);
}
// If we got here, we probably have good data
// Make AddonInternal instances from the loaded data and save them
let addonDB = new Map();
inputAddons.addons.forEach(function(loadedAddon) {
let newAddon = new DBAddonInternal(loadedAddon);
addonDB.set(newAddon._key, newAddon);
});
this.addonDB = addonDB;
LOG("Successfully read XPI database");
}
catch(e) {
// If we catch and log a SyntaxError from the JSON
// parser, the xpcshell test harness fails the test for us: bug 870828
if (e.name == "SyntaxError") {
ERROR("Syntax error parsing saved XPI JSON data");
}
else {
ERROR("Failed to load XPI JSON data from profile", e);
}
this.rebuildDatabase(aRebuildOnError);
}
finally {
if (cstream)
cstream.close();
}
}
catch (e) {
if (e.result == Cr.NS_ERROR_FILE_NOT_FOUND) {
// XXX re-implement logic to decide whether to upgrade database
// by checking the DB_SCHEMA_VERSION preference.
// Fall back to attempting database upgrades
WARN("Extensions database not found; attempting to upgrade");
// See if there is SQLITE to migrate from
if (!this.loadSqliteData()) {
// Nope, try RDF
this.migrateData = this.getMigrateDataFromRDF();
}
this.rebuildDatabase(aRebuildOnError);
}
else {
WARN("Extensions database " + this.jsonFile.path +
" exists but is not readable; rebuilding in memory", e);
// XXX open question - if we can overwrite at save time, should we, or should we
// leave the locked database in case we can recover from it next time we start up?
this.lockedDatabase = true;
// XXX TELEMETRY report when this happens?
this.rebuildDatabase(aRebuildOnError);
}
}
finally {
if (fstream)
fstream.close();
}
this.initialized = true;
return;
// XXX IRVING deal with the migration logic below and in openDatabaseFile...
delete this.connection;
// XXX what about aForceOpen? Appears to handle the case of "don't open DB file if there aren't any extensions"?
if (!aForceOpen && !this.dbfileExists) {
this.connection = null;
return;
}
},
this.migrateData = null;
/**
* Rebuild the database from addon install directories. If this.migrateData
* is available, uses migrated information for settings on the addons found
* during rebuild
* @param aRebuildOnError
* A boolean indicating whether add-on information should be loaded
* from the install locations if the database needs to be rebuilt.
* (if false, caller is XPIProvider.checkForChanges() which will rebuild)
*/
rebuildDatabase: function XIPDB_rebuildDatabase(aRebuildOnError) {
// If there is no migration data then load the list of add-on directories
// that were active during the last run
this.addonDB = new Map();
if (!this.migrateData)
this.activeBundles = this.getActiveBundles();
this.connection = this.openDatabaseFile(this.dbfile);
// If the database was corrupt or missing then the new blank database will
// have a schema version of 0.
let schemaVersion = this.connection.schemaVersion;
if (schemaVersion != DB_SCHEMA) {
// A non-zero schema version means that a schema has been successfully
// created in the database in the past so we might be able to get useful
// information from it
if (schemaVersion != 0) {
LOG("Migrating data from schema " + schemaVersion);
this.migrateData = this.getMigrateDataFromDatabase();
// Delete the existing database
this.connection.close();
try {
if (this.dbfileExists)
this.dbfile.remove(true);
// Reopen an empty database
this.connection = this.openDatabaseFile(this.dbfile);
}
catch (e) {
ERROR("Failed to remove old database", e);
// If the file couldn't be deleted then fall back to an in-memory
// database
this.connection = Services.storage.openSpecialDatabase("memory");
}
}
else {
let dbSchema = 0;
try {
dbSchema = Services.prefs.getIntPref(PREF_DB_SCHEMA);
} catch (e) {}
if (dbSchema == 0) {
// Only migrate data from the RDF if we haven't done it before
this.migrateData = this.getMigrateDataFromRDF();
}
}
// At this point the database should be completely empty
if (aRebuildOnError) {
WARN("Rebuilding add-ons database from installed extensions.");
this.beginTransaction();
try {
this.createSchema();
let state = XPIProvider.getInstallLocationStates();
XPIProvider.processFileChanges(state, {}, false);
// Make sure to update the active add-ons and add-ons list on shutdown
Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true);
this.commitTransaction();
}
catch (e) {
// If creating the schema fails, then the database is unusable,
// fall back to an in-memory database.
this.connection = Services.storage.openSpecialDatabase("memory");
}
// If there is no migration data then load the list of add-on directories
// that were active during the last run
if (!this.migrateData)
this.activeBundles = this.getActiveBundles();
if (aRebuildOnError) {
WARN("Rebuilding add-ons database from installed extensions.");
this.beginTransaction();
try {
let state = XPIProvider.getInstallLocationStates();
XPIProvider.processFileChanges(state, {}, false);
// Make sure to update the active add-ons and add-ons list on shutdown
Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true);
this.commitTransaction();
}
catch (e) {
ERROR("Error processing file changes", e);
this.rollbackTransaction();
}
ERROR("Error processing file changes", e);
this.rollbackTransaction();
}
}
// If the database connection has a file open then it has the right schema
// by now so make sure the preferences reflect that.
if (this.connection.databaseFile) {
Services.prefs.setIntPref(PREF_DB_SCHEMA, DB_SCHEMA);
Services.prefs.savePrefFile(null);
}
// Begin any pending transactions
for (let i = 0; i < this.transactionCount; i++)
this.connection.executeSimpleSQL("SAVEPOINT 'default'");
},
/**
* Lazy getter for the addons database
*/
get addonDB() {
delete this.addonDB;
this.openJSONDatabase();
this.openConnection(true);
return this.addonDB;
},
@ -964,13 +794,13 @@ this.XPIDatabase = {
* @return an object holding information about what add-ons were previously
* userDisabled and any updated compatibility information
*/
getMigrateDataFromDatabase: function XPIDB_getMigrateDataFromDatabase() {
getMigrateDataFromDatabase: function XPIDB_getMigrateDataFromDatabase(aConnection) {
let migrateData = {};
// Attempt to migrate data from a different (even future!) version of the
// database
try {
var stmt = this.connection.createStatement("PRAGMA table_info(addon)");
var stmt = aConnection.createStatement("PRAGMA table_info(addon)");
const REQUIRED = ["internal_id", "id", "location", "userDisabled",
"installDate", "version"];
@ -996,7 +826,7 @@ this.XPIDatabase = {
}
stmt.finalize();
stmt = this.connection.createStatement("SELECT " + props.join(",") + " FROM addon");
stmt = aConnection.createStatement("SELECT " + props.join(",") + " FROM addon");
for (let row in resultRows(stmt)) {
if (!(row.location in migrateData))
migrateData[row.location] = {};
@ -1015,7 +845,7 @@ this.XPIDatabase = {
})
}
var taStmt = this.connection.createStatement("SELECT id, minVersion, " +
var taStmt = aConnection.createStatement("SELECT id, minVersion, " +
"maxVersion FROM " +
"targetApplication WHERE " +
"addon_internal_id=:internal_id");
@ -1063,10 +893,8 @@ this.XPIDatabase = {
// If we are running with an in-memory database then force a new
// extensions.ini to be written to disk on the next startup
// XXX IRVING special case for if we fail to save extensions.json?
// XXX maybe doesn't need to be at shutdown?
// if (!this.connection.databaseFile)
// Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true);
if (this.lockedDatabase)
Services.prefs.setBoolPref(PREF_PENDING_OPERATIONS, true);
this.initialized = false;
@ -1075,7 +903,7 @@ this.XPIDatabase = {
delete this.addonDB;
Object.defineProperty(this, "addonDB", {
get: function addonsGetter() {
this.openJSONDatabase();
this.openConnection(true);
return this.addonDB;
},
configurable: true
@ -1098,17 +926,17 @@ this.XPIDatabase = {
* installed add-ons, occasionally a superset when an install location no
* longer exists.
*
* @return an array of names of install locations
* @return a Set of names of install locations
*/
getInstallLocations: function XPIDB_getInstallLocations() {
let locations = new Set();
if (!this.addonDB)
return [];
return locations;
let locations = {};
for each (let addon in this.addonDB) {
locations[addon.location] = 1;
for (let [, addon] of this.addonDB) {
locations.add(addon.location);
}
return Object.keys(locations);
return locations;
},
/**
@ -1123,8 +951,7 @@ this.XPIDatabase = {
return [];
let addonList = [];
for (let key in this.addonDB) {
let addon = this.addonDB[key];
for (let [key, addon] of this.addonDB) {
if (aFilter(addon)) {
addonList.push(addon);
}
@ -1144,8 +971,7 @@ this.XPIDatabase = {
if (!this.addonDB)
return null;
for (let key in this.addonDB) {
let addon = this.addonDB[key];
for (let [key, addon] of this.addonDB) {
if (aFilter(addon)) {
return addon;
}
@ -1178,7 +1004,7 @@ this.XPIDatabase = {
* A callback to pass the DBAddonInternal to
*/
getAddonInLocation: function XPIDB_getAddonInLocation(aId, aLocation, aCallback) {
getRepositoryAddon(this.addonDB[aLocation + ":" + aId], aCallback);
getRepositoryAddon(this.addonDB.get(aLocation + ":" + aId), aCallback);
},
/**
@ -1305,7 +1131,7 @@ this.XPIDatabase = {
let newAddon = new DBAddonInternal(aAddon);
newAddon.descriptor = aDescriptor;
this.addonDB[newAddon._key] = newAddon;
this.addonDB.set(newAddon._key, newAddon);
if (newAddon.visible) {
this.makeAddonVisible(newAddon);
}
@ -1358,7 +1184,7 @@ this.XPIDatabase = {
*/
removeAddonMetadata: function XPIDB_removeAddonMetadata(aAddon) {
this.beginTransaction();
delete this.addonDB[aAddon._key];
this.addonDB.delete(aAddon._key);
this.commitTransaction();
},
@ -1374,8 +1200,7 @@ this.XPIDatabase = {
makeAddonVisible: function XPIDB_makeAddonVisible(aAddon) {
this.beginTransaction();
LOG("Make addon " + aAddon._key + " visible");
for (let key in this.addonDB) {
let otherAddon = this.addonDB[key];
for (let [key, otherAddon] of this.addonDB) {
if ((otherAddon.id == aAddon.id) && (otherAddon._key != aAddon._key)) {
LOG("Hide addon " + otherAddon._key);
otherAddon.visible = false;
@ -1461,14 +1286,20 @@ this.XPIDatabase = {
// XXX IRVING this may get called during XPI-utils shutdown
// XXX need to make sure PREF_PENDING_OPERATIONS handling is clean
LOG("Updating add-on states");
this.beginTransaction();
for (let key in this.addonDB) {
let addon = this.addonDB[key];
addon.active = (addon.visible && !addon.userDisabled &&
let changed = false;
for (let [key, addon] of this.addonDB) {
let newActive = (addon.visible && !addon.userDisabled &&
!addon.softDisabled && !addon.appDisabled &&
!addon.pendingUninstall);
if (newActive != addon.active) {
addon.active = newActive;
changed = true;
}
}
if (changed) {
this.beginTransaction();
this.commitTransaction();
}
this.commitTransaction();
},
/**

10
toolkit/mozapps/extensions/test/xpcshell/head_addons.js
View File

@ -1395,16 +1395,16 @@ function do_exception_wrap(func) {
}
const EXTENSIONS_DB = "extensions.json";
let gExtensionsJSON = gProfD.clone();
gExtensionsJSON.append(EXTENSIONS_DB);
/**
* Change the schema version of the JSON extensions database
*/
function changeXPIDBVersion(aNewVersion) {
let dbfile = gProfD.clone();
dbfile.append(EXTENSIONS_DB);
let jData = loadJSON(dbfile);
let jData = loadJSON(gExtensionsJSON);
jData.schemaVersion = aNewVersion;
saveJSON(jData, dbfile);
saveJSON(jData, gExtensionsJSON);
}
/**
@ -1426,7 +1426,7 @@ function loadJSON(aFile) {
} while (read != 0);
}
cstream.close();
do_print("Loaded JSON file " + aFile.spec);
do_print("Loaded JSON file " + aFile.path);
return(JSON.parse(data));
}

54
toolkit/mozapps/extensions/test/xpcshell/test_bad_json.js Normal file
View File

@ -0,0 +1,54 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
// Tests that we rebuild the database correctly if it contains
// JSON data that parses correctly but doesn't contain required fields
var addon1 = {
id: "addon1@tests.mozilla.org",
version: "2.0",
name: "Test 1",
targetApplications: [{
id: "xpcshell@tests.mozilla.org",
minVersion: "1",
maxVersion: "1"
}]
};
const profileDir = gProfD.clone();
profileDir.append("extensions");
function run_test() {
do_test_pending("Bad JSON");
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
// This addon will be auto-installed at startup
writeInstallRDFForExtension(addon1, profileDir);
startupManager();
shutdownManager();
// First startup/shutdown finished
// Replace the JSON store with something bogus
saveJSON({not: "what we expect to find"}, gExtensionsJSON);
startupManager(false);
// Retrieve an addon to force the database to rebuild
AddonManager.getAddonsByIDs([addon1.id], callback_soon(after_db_rebuild));
}
function after_db_rebuild([a1]) {
do_check_eq(a1.id, addon1.id);
shutdownManager();
// Make sure our JSON database has schemaVersion and our installed extension
let data = loadJSON(gExtensionsJSON);
do_check_true("schemaVersion" in data);
do_check_eq(data.addons[0].id, addon1.id);
do_test_finished("Bad JSON");
}

10
toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js
View File

@ -146,10 +146,9 @@ function run_test() {
startupManager();
let file = gProfD.clone();
file.append(EXTENSIONS_DB);
do_check_false(file.exists());
do_check_false(gExtensionsJSON.exists());
let file = gProfD.clone();
file.leafName = "extensions.ini";
do_check_false(file.exists());
@ -205,10 +204,9 @@ function run_test_1() {
}
function check_test_1(installSyncGUID) {
let file = gProfD.clone();
file.append(EXTENSIONS_DB);
do_check_true(file.exists());
do_check_true(gExtensionsJSON.exists());
let file = gProfD.clone();
file.leafName = "extensions.ini";
do_check_false(file.exists());

10
toolkit/mozapps/extensions/test/xpcshell/test_bug559800.js
View File

@ -47,9 +47,7 @@ function run_test_1() {
shutdownManager();
let db = gProfD.clone();
db.append(EXTENSIONS_DB);
db.remove(true);
gExtensionsJSON.remove(true);
do_execute_soon(check_test_1);
});
@ -62,10 +60,8 @@ function check_test_1() {
do_check_neq(a1, null);
do_check_eq(a1.version, "1.0");
let db = gProfD.clone();
db.append(EXTENSIONS_DB);
do_check_true(db.exists());
do_check_true(db.fileSize > 0);
do_check_true(gExtensionsJSON.exists());
do_check_true(gExtensionsJSON.fileSize > 0);
end_test();
});

6
toolkit/mozapps/extensions/test/xpcshell/test_corrupt.js
View File

@ -251,10 +251,8 @@ function run_test_1() {
// serves this purpose). On startup the add-ons manager won't rebuild
// because there is a file there still.
shutdownManager();
var dbfile = gProfD.clone();
dbfile.append("extensions.json");
dbfile.remove(true);
dbfile.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755);
gExtensionsJSON.remove(true);
gExtensionsJSON.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
startupManager(false);
// Accessing the add-ons should open and recover the database

6
toolkit/mozapps/extensions/test/xpcshell/test_corrupt_strictcompat.js
View File

@ -252,10 +252,8 @@ function run_test_1() {
// serves this purpose). On startup the add-ons manager won't rebuild
// because there is a file there still.
shutdownManager();
var dbfile = gProfD.clone();
dbfile.append("extensions.json");
dbfile.remove(true);
dbfile.create(AM_Ci.nsIFile.DIRECTORY_TYPE, 0755);
gExtensionsJSON.remove(true);
gExtensionsJSON.create(AM_Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
startupManager(false);
// Accessing the add-ons should open and recover the database

13
toolkit/mozapps/extensions/test/xpcshell/test_locked.js
View File

@ -257,10 +257,8 @@ function run_test_1() {
// After shutting down the database won't be open so we can
// mess with permissions
shutdownManager();
var dbfile = gProfD.clone();
dbfile.append(EXTENSIONS_DB);
var savedPermissions = dbfile.permissions;
dbfile.permissions = 0;
var savedPermissions = gExtensionsJSON.permissions;
gExtensionsJSON.permissions = 0;
startupManager(false);
@ -428,11 +426,12 @@ function run_test_1() {
do_check_eq(t2.pendingOperations, AddonManager.PENDING_NONE);
do_check_true(isThemeInAddonsList(profileDir, t2.id));
dbfile.permissions = savedPermissions;
// After allowing access to the original DB things should go back to as
// they were previously
restartManager();
shutdownManager();
gExtensionsJSON.permissions = savedPermissions;
startupManager();
// Shouldn't have seen any startup changes
check_startup_changes(AddonManager.STARTUP_CHANGE_INSTALLED, []);

12
toolkit/mozapps/extensions/test/xpcshell/test_locked2.js
View File

@ -144,10 +144,8 @@ function run_test() {
// After shutting down the database won't be open so we can lock it
shutdownManager();
var dbfile = gProfD.clone();
dbfile.append(EXTENSIONS_DB);
var savedPermissions = dbfile.permissions;
dbfile.permissions = 0;
var savedPermissions = gExtensionsJSON.permissions;
gExtensionsJSON.permissions = 0;
startupManager(false);
@ -199,11 +197,11 @@ function run_test() {
do_check_eq(a6.pendingOperations, AddonManager.PENDING_NONE);
do_check_true(isExtensionInAddonsList(profileDir, a6.id));
dbfile.permissions = savedPermissions;
// After allowing access to the original DB things should still be
// applied correctly
restartManager();
shutdownManager();
gExtensionsJSON.permissions = savedPermissions;
startupManager();
// These things happened when we had no access to the database so
// they are seen as external changes when we get the database back :(

12
toolkit/mozapps/extensions/test/xpcshell/test_locked_strictcompat.js
View File

@ -256,10 +256,8 @@ function run_test_1() {
// After shutting down the database won't be open so we can lock it
shutdownManager();
var dbfile = gProfD.clone();
dbfile.append(EXTENSIONS_DB);
var savedPermissions = dbfile.permissions;
dbfile.permissions = 0;
var savedPermissions = gExtensionsJSON.permissions;
gExtensionsJSON.permissions = 0;
startupManager(false);
@ -425,11 +423,11 @@ function run_test_1() {
do_check_eq(t2.pendingOperations, AddonManager.PENDING_NONE);
do_check_true(isThemeInAddonsList(profileDir, t2.id));
dbfile.permissions = savedPermissions;
// After allowing access to the original DB things should go back to as
// they were previously
restartManager();
shutdownManager();
gExtensionsJSON.permissions = savedPermissions;
startupManager(false);
// Shouldn't have seen any startup changes
check_startup_changes(AddonManager.STARTUP_CHANGE_INSTALLED, []);

27
toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
View File

@ -16,12 +16,9 @@ skip-if = os == "android"
[test_DeferredSave.js]
[test_LightweightThemeManager.js]
[test_backgroundupdate.js]
[test_bad_json.js]
[test_badschema.js]
# Needs rewrite for JSON XPIDB
fail-if = true
[test_blocklistchange.js]
# Needs rewrite for JSON XPIDB
fail-if = true
# Bug 676992: test consistently hangs on Android
skip-if = os == "android"
[test_blocklist_regexp.js]
@ -142,8 +139,6 @@ fail-if = os == "android"
[test_bug620837.js]
[test_bug655254.js]
[test_bug659772.js]
# needs to be converted from sqlite to JSON
fail-if = true
[test_bug675371.js]
[test_bug740612.js]
[test_bug753900.js]
@ -153,11 +148,7 @@ fail-if = true
[test_ChromeManifestParser.js]
[test_compatoverrides.js]
[test_corrupt.js]
# needs to be converted from sqlite to JSON
fail-if = true
[test_corrupt_strictcompat.js]
# needs to be converted from sqlite to JSON
fail-if = true
[test_dictionary.js]
[test_langpack.js]
[test_disable.js]
@ -202,33 +193,17 @@ skip-if = os == "android"
run-sequentially = Uses hardcoded ports in xpi files.
[test_locale.js]
[test_locked.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_locked2.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_locked_strictcompat.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_manifest.js]
[test_mapURIToAddonID.js]
# Same as test_bootstrap.js
skip-if = os == "android"
[test_migrate1.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_migrate2.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_migrate3.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_migrate4.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_migrate5.js]
# Needs sqlite->JSON conversion
fail-if = true
[test_migrateAddonRepository.js]
[test_onPropertyChanged_appDisabled.js]
[test_permissions.js]