/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ /** * Managing safe shutdown of asynchronous services. * * Firefox shutdown is composed of phases that take place * sequentially. Typically, each shutdown phase removes some * capabilities from the application. For instance, at the end of * phase profileBeforeChange, no service is permitted to write to the * profile directory (with the exception of Telemetry). Consequently, * if any service has requested I/O to the profile directory before or * during phase profileBeforeChange, the system must be informed that * these requests need to be completed before the end of phase * profileBeforeChange. Failing to inform the system of this * requirement can (and has been known to) cause data loss. * * Example: At some point during shutdown, the Add-On Manager needs to * ensure that all add-ons have safely written their data to disk, * before writing its own data. Since the data is saved to the * profile, this must be completed during phase profileBeforeChange. * * AsyncShutdown.profileBeforeChange.addBlocker( * "Add-on manager: shutting down", * function condition() { * // Do things. * // Perform I/O that must take place during phase profile-before-change * return promise; * } * }); * * In this example, function |condition| will be called at some point * during phase profileBeforeChange and phase profileBeforeChange * itself is guaranteed to not terminate until |promise| is either * resolved or rejected. */ "use strict"; const Cu = Components.utils; const Cc = Components.classes; const Ci = Components.interfaces; Cu.import("resource://gre/modules/XPCOMUtils.jsm", this); Cu.import("resource://gre/modules/Services.jsm", this); XPCOMUtils.defineLazyModuleGetter(this, "Promise", "resource://gre/modules/Promise.jsm"); XPCOMUtils.defineLazyModuleGetter(this, "Task", "resource://gre/modules/Task.jsm"); XPCOMUtils.defineLazyServiceGetter(this, "gDebug", "@mozilla.org/xpcom/debug;1", "nsIDebug"); Object.defineProperty(this, "gCrashReporter", { get: function() { delete this.gCrashReporter; try { let reporter = Cc["@mozilla.org/xre/app-info;1"]. getService(Ci.nsICrashReporter); return this.gCrashReporter = reporter; } catch (ex) { return this.gCrashReporter = null; } }, configurable: true }); // Display timeout warnings after 10 seconds const DELAY_WARNING_MS = 10 * 1000; // Crash the process if shutdown is really too long // (allowing for sleep). const PREF_DELAY_CRASH_MS = "toolkit.asyncshutdown.crash_timeout"; let DELAY_CRASH_MS = 60 * 1000; // One minute try { DELAY_CRASH_MS = Services.prefs.getIntPref(PREF_DELAY_CRASH_MS); } catch (ex) { // Ignore errors } Services.prefs.addObserver(PREF_DELAY_CRASH_MS, function() { DELAY_CRASH_MS = Services.prefs.getIntPref(PREF_DELAY_CRASH_MS); }, false); /** * Display a warning. * * As this code is generally used during shutdown, there are chances * that the UX will not be available to display warnings on the * console. We therefore use dump() rather than Cu.reportError(). */ function log(msg, prefix = "", error = null) { dump(prefix + msg + "\n"); if (error) { dump(prefix + error + "\n"); if (typeof error == "object" && "stack" in error) { dump(prefix + error.stack + "\n"); } } } function warn(msg, error = null) { return log(msg, "WARNING: ", error); } function fatalerr(msg, error = null) { return log(msg, "FATAL ERROR: ", error); } // Utility function designed to get the current state of execution // of a blocker. // We are a little paranoid here to ensure that in case of evaluation // error we do not block the AsyncShutdown. function safeGetState(fetchState) { if (!fetchState) { return "(none)"; } let data, string; try { // Evaluate fetchState(), normalize the result into something that we can // safely stringify or upload. string = JSON.stringify(fetchState()); data = JSON.parse(string); // Simplify the rest of the code by ensuring that we can simply // concatenate the result to a message. if (data && typeof data == "object") { data.toString = function() { return string; }; } return data; } catch (ex) { if (string) { return string; } try { return "Error getting state: " + ex + " at " + ex.stack; } catch (ex2) { return "Error getting state but could not display error"; } } } /** * Countdown for a given duration, skipping beats if the computer is too busy, * sleeping or otherwise unavailable. * * @param {number} delay An approximate delay to wait in milliseconds (rounded * up to the closest second). * * @return Deferred */ function looseTimer(delay) { let DELAY_BEAT = 1000; let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); let beats = Math.ceil(delay / DELAY_BEAT); let deferred = Promise.defer(); timer.initWithCallback(function() { if (beats <= 0) { deferred.resolve(); } --beats; }, DELAY_BEAT, Ci.nsITimer.TYPE_REPEATING_PRECISE_CAN_SKIP); // Ensure that the timer is both canceled once we are done with it // and not garbage-collected until then. deferred.promise.then(() => timer.cancel(), () => timer.cancel()); return deferred; } this.EXPORTED_SYMBOLS = ["AsyncShutdown"]; /** * {string} topic -> phase */ let gPhases = new Map(); this.AsyncShutdown = { /** * Access function getPhase. For testing purposes only. */ get _getPhase() { let accepted = false; try { accepted = Services.prefs.getBoolPref("toolkit.asyncshutdown.testing"); } catch (ex) { // Ignore errors } if (accepted) { return getPhase; } return undefined; } }; /** * Register a new phase. * * @param {string} topic The notification topic for this Phase. * @see {https://developer.mozilla.org/en-US/docs/Observer_Notifications} */ function getPhase(topic) { let phase = gPhases.get(topic); if (phase) { return phase; } let spinner = new Spinner(topic); phase = Object.freeze({ /** * Register a blocker for the completion of a phase. * * @param {string} name The human-readable name of the blocker. Used * for debugging/error reporting. Please make sure that the name * respects the following model: "Some Service: some action in progress" - * for instance "OS.File: flushing all pending I/O"; * @param {function|promise|*} condition A condition blocking the * completion of the phase. Generally, this is a function * returning a promise. This function is evaluated during the * phase and the phase is guaranteed to not terminate until the * resulting promise is either resolved or rejected. If * |condition| is not a function but another value |v|, it behaves * as if it were a function returning |v|. * @param {function*} fetchState Optionally, a function returning * information about the current state of the blocker as an * object. Used for providing more details when logging errors or * crashing. * * Examples: * AsyncShutdown.profileBeforeChange.addBlocker("Module: just a promise", * promise); // profileBeforeChange will not complete until * // promise is resolved or rejected * * AsyncShutdown.profileBeforeChange.addBlocker("Module: a callback", * function callback() { * // ... * // Execute this code during profileBeforeChange * return promise; * // profileBeforeChange will not complete until promise * // is resolved or rejected * }); * * AsyncShutdown.profileBeforeChange.addBlocker("Module: trivial callback", * function callback() { * // ... * // Execute this code during profileBeforeChange * // No specific guarantee about completion of profileBeforeChange * }); */ addBlocker: function(name, condition, fetchState = null) { spinner.addBlocker(name, condition, fetchState); }, /** * Remove the blocker for a condition. * * If several blockers have been registered for the same * condition, remove all these blockers. If no blocker has been * registered for this condition, this is a noop. * * @return {boolean} true if a blocker has been removed, false * otherwise. Note that a result of false may mean either that * the blocker has never been installed or that the phase has * completed and the blocker has already been resolved. */ removeBlocker: function(condition) { return spinner.removeBlocker(condition); }, /** * Trigger the phase without having to broadcast a * notification. For testing purposes only. */ get _trigger() { let accepted = false; try { accepted = Services.prefs.getBoolPref("toolkit.asyncshutdown.testing"); } catch (ex) { // Ignore errors } if (accepted) { return () => spinner.observe(); } return undefined; } }); gPhases.set(topic, phase); return phase; }; /** * Utility class used to spin the event loop until all blockers for a * Phase are satisfied. * * @param {string} topic The xpcom notification for that phase. */ function Spinner(topic) { this._barrier = new Barrier(topic); this._topic = topic; Services.obs.addObserver(this, topic, false); } Spinner.prototype = { /** * Register a new condition for this phase. * * @param {object} condition A Condition that must be fulfilled before * we complete this Phase. * Must contain fields: * - {string} name The human-readable name of the condition. Used * for debugging/error reporting. * - {function} action An action that needs to be completed * before we proceed to the next runstate. If |action| returns a promise, * we wait until the promise is resolved/rejected before proceeding * to the next runstate. */ addBlocker: function(name, condition, fetchState) { this._barrier.client.addBlocker(name, condition, fetchState); }, /** * Remove the blocker for a condition. * * If several blockers have been registered for the same * condition, remove all these blockers. If no blocker has been * registered for this condition, this is a noop. * * @return {boolean} true if a blocker has been removed, false * otherwise. Note that a result of false may mean either that * the blocker has never been installed or that the phase has * completed and the blocker has already been resolved. */ removeBlocker: function(condition) { return this._barrier.client.removeBlocker(condition); }, // nsIObserver.observe observe: function() { let topic = this._topic; let barrier = this._barrier; Services.obs.removeObserver(this, topic); let satisfied = false; // |true| once we have satisfied all conditions let promise = this._barrier.wait({ warnAfterMS: DELAY_WARNING_MS, crashAfterMS: DELAY_CRASH_MS }); // Now, spin the event loop promise.then(() => satisfied = true); // This promise cannot reject let thread = Services.tm.mainThread; while (!satisfied) { try { thread.processNextEvent(true); } catch (ex) { // An uncaught error should not stop us, but it should still // be reported and cause tests to fail. Promise.reject(ex); } } } }; /** * A mechanism used to register blockers that prevent some action from * happening. * * An instance of |Barrier| provides a capability |client| that * clients can use to register blockers. The barrier is resolved once * all registered blockers have been resolved. The owner of the * |Barrier| may wait for the resolution of the barrier and obtain * information on which blockers have not been resolved yet. * * @param {string} name The name of the blocker. Used mainly for error- * reporting. */ function Barrier(name) { /** * The set of conditions registered by clients, as a map. * * Key: condition (function) * Value: Array of {name: string, fetchState: function} */ this._conditions = new Map(); /** * Indirections, used to let clients cancel a blocker when they * call removeBlocker(). * * Key: condition (function) * Value: Deferred. */ this._indirections = null; /** * The name of the barrier. */ this._name = name; /** * A cache for the promise returned by wait(). */ this._promise = null; /** * An array of objects used to monitor the state of each blocker. */ this._monitors = null; /** * The capability of adding blockers. This object may safely be returned * or passed to clients. */ this.client = { /** * Register a blocker for the completion of this barrier. * * @param {string} name The human-readable name of the blocker. Used * for debugging/error reporting. Please make sure that the name * respects the following model: "Some Service: some action in progress" - * for instance "OS.File: flushing all pending I/O"; * @param {function|promise|*} condition A condition blocking the * completion of the phase. Generally, this is a function * returning a promise. This function is evaluated during the * phase and the phase is guaranteed to not terminate until the * resulting promise is either resolved or rejected. If * |condition| is not a function but another value |v|, it behaves * as if it were a function returning |v|. * @param {function*} fetchState Optionally, a function returning * information about the current state of the blocker as an * object. Used for providing more details when logging errors or * crashing. */ addBlocker: function(name, condition, fetchState) { if (typeof name != "string") { throw new TypeError("Expected a human-readable name as first argument"); } if (fetchState && typeof fetchState != "function") { throw new TypeError("Expected nothing or a function as third argument"); } if (!this._conditions) { throw new Error("Phase " + this._name + " has already begun, it is too late to register" + " completion condition '" + name + "'."); } // Determine the filename and line number of the caller. let leaf = Components.stack; let frame; for (frame = leaf; frame != null && frame.filename == leaf.filename; frame = frame.caller) { // Climb up the stack } let filename = frame ? frame.filename : "?"; let lineNumber = frame ? frame.lineNumber : -1; // Now build the rest of the stack as a string, using Task.jsm's rewriting // to ensure that we do not lose information at each call to `Task.spawn`. let frames = []; while (frame != null) { frames.push(frame.filename + ":" + frame.name + ":" + frame.lineNumber); frame = frame.caller; } let stack = Task.Debugging.generateReadableStack(frames.join("\n")).split("\n"); let set = this._conditions.get(condition); if (!set) { set = []; this._conditions.set(condition, set); } set.push({name: name, fetchState: fetchState, filename: filename, lineNumber: lineNumber, stack: stack}); }.bind(this), /** * Remove the blocker for a condition. * * If several blockers have been registered for the same * condition, remove all these blockers. If no blocker has been * registered for this condition, this is a noop. * * @return {boolean} true if at least one blocker has been * removed, false otherwise. */ removeBlocker: function(condition) { if (this._conditions) { // wait() hasn't been called yet. return this._conditions.delete(condition); } if (this._indirections) { // wait() is in progress let deferred = this._indirections.get(condition); if (deferred) { // Unlock the blocker deferred.resolve(); } return this._indirections.delete(condition); } // wait() is complete. return false; }.bind(this), }; } Barrier.prototype = Object.freeze({ /** * The current state of the barrier, as a JSON-serializable object * designed for error-reporting. */ get state() { if (this._conditions) { return "Not started"; } if (!this._monitors) { return "Complete"; } let frozen = []; for (let {name, isComplete, fetchState, stack, filename, lineNumber} of this._monitors) { if (!isComplete) { frozen.push({name: name, state: safeGetState(fetchState), filename: filename, lineNumber: lineNumber, stack: stack}); } } return frozen; }, /** * Wait until all currently registered blockers are complete. * * Once this method has been called, any attempt to register a new blocker * for this barrier will cause an error. * * Successive calls to this method always return the same value. * * @param {object=} options Optionally, an object that may contain * the following fields: * {number} warnAfterMS If provided and > 0, print a warning if the barrier * has not been resolved after the given number of milliseconds. * {number} crashAfterMS If provided and > 0, crash the process if the barrier * has not been resolved after the give number of milliseconds (rounded up * to the next second). To avoid crashing simply because the computer is busy * or going to sleep, we actually wait for ceil(crashAfterMS/1000) successive * periods of at least one second. Upon crashing, if a crash reporter is present, * prepare a crash report with the state of this barrier. * * * @return {Promise} A promise satisfied once all blockers are complete. */ wait: function(options = {}) { // This method only implements caching on top of _wait() if (this._promise) { return this._promise; } return this._promise = this._wait(options); }, _wait: function(options) { let topic = this._name; let conditions = this._conditions; this._conditions = null; // Too late to register if (conditions.size == 0) { return Promise.resolve(); } this._indirections = new Map(); // The promises for which we are waiting. let allPromises = []; // Information to determine and report to the user which conditions // are not satisfied yet. this._monitors = []; for (let _condition of conditions.keys()) { for (let current of conditions.get(_condition)) { let condition = _condition; // Avoid capturing the wrong variable let {name, fetchState, stack, filename, lineNumber} = current; // An indirection on top of condition, used to let clients // cancel a blocker through removeBlocker. let indirection = Promise.defer(); this._indirections.set(condition, indirection); // Gather all completion conditions try { if (typeof condition == "function") { // Normalize |condition| to the result of the function. try { condition = condition(topic); } catch (ex) { condition = Promise.reject(ex); } } // Normalize to a promise. Of course, if |condition| was not a // promise in the first place (in particular if the above // function returned |undefined| or failed), that new promise // isn't going to be terribly interesting, but it will behave // as a promise. condition = Promise.resolve(condition); let monitor = { isComplete: false, name: name, fetchState: fetchState, stack: stack, filename: filename, lineNumber: lineNumber }; condition = condition.then(null, function onError(error) { let msg = "A completion condition encountered an error" + " while we were spinning the event loop." + " Condition: " + name + " Phase: " + topic + " State: " + safeGetState(fetchState); warn(msg, error); // The error should remain uncaught, to ensure that it // still causes tests to fail. Promise.reject(error); }); condition.then(() => indirection.resolve()); indirection.promise.then(() => monitor.isComplete = true); this._monitors.push(monitor); allPromises.push(indirection.promise); } catch (error) { let msg = "A completion condition encountered an error" + " while we were initializing the phase." + " Condition: " + name + " Phase: " + topic + " State: " + safeGetState(fetchState); warn(msg, error); } } } conditions = null; let promise = Promise.all(allPromises); allPromises = null; promise = promise.then(null, function onError(error) { // I don't think that this can happen. // However, let's be overcautious with async/shutdown error reporting. let msg = "An uncaught error appeared while completing the phase." + " Phase: " + topic; warn(msg, error); }); promise = promise.then(() => { this._monitors = null; this._indirections = null; }); // Memory cleanup // Now handle warnings and crashes let warnAfterMS = DELAY_WARNING_MS; if (options && "warnAfterMS" in options) { if (typeof options.warnAfterMS == "number" || options.warnAfterMS == null) { // Change the delay or deactivate warnAfterMS warnAfterMS = options.warnAfterMS; } else { throw new TypeError("Wrong option value for warnAfterMS"); } } if (warnAfterMS && warnAfterMS > 0) { // If the promise takes too long to be resolved/rejected, // we need to notify the user. let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); timer.initWithCallback(function() { let msg = "At least one completion condition is taking too long to complete." + " Conditions: " + JSON.stringify(this.state) + " Barrier: " + topic; warn(msg); }.bind(this), warnAfterMS, Ci.nsITimer.TYPE_ONE_SHOT); promise = promise.then(function onSuccess() { timer.cancel(); // As a side-effect, this prevents |timer| from // being garbage-collected too early. }); } let crashAfterMS = DELAY_CRASH_MS; if (options && "crashAfterMS" in options) { if (typeof options.crashAfterMS == "number" || options.crashAfterMS == null) { // Change the delay or deactivate crashAfterMS crashAfterMS = options.crashAfterMS; } else { throw new TypeError("Wrong option value for crashAfterMS"); } } if (crashAfterMS > 0) { let timeToCrash = null; // If after |crashAfterMS| milliseconds (adjusted to take into // account sleep and otherwise busy computer) we have not finished // this shutdown phase, we assume that the shutdown is somehow // frozen, presumably deadlocked. At this stage, the only thing we // can do to avoid leaving the user's computer in an unstable (and // battery-sucking) situation is report the issue and crash. timeToCrash = looseTimer(crashAfterMS); timeToCrash.promise.then( function onTimeout() { // Report the problem as best as we can, then crash. let state = this.state; // If you change the following message, please make sure // that any information on the topic and state appears // within the first 200 characters of the message. This // helps automatically sort oranges. let msg = "AsyncShutdown timeout in " + topic + " Conditions: " + JSON.stringify(state) + " At least one completion condition failed to complete" + " within a reasonable amount of time. Causing a crash to" + " ensure that we do not leave the user with an unresponsive" + " process draining resources."; fatalerr(msg); if (gCrashReporter && gCrashReporter.enabled) { let data = { phase: topic, conditions: state }; gCrashReporter.annotateCrashReport("AsyncShutdownTimeout", JSON.stringify(data)); } else { warn("No crash reporter available"); } // To help sorting out bugs, we want to make sure that the // call to nsIDebug.abort points to a guilty client, rather // than to AsyncShutdown itself. We search through all the // clients until we find one that is guilty and use its // filename/lineNumber, which have been determined during // the call to `addBlocker`. let filename = "?"; let lineNumber = -1; for (let monitor of this._monitors) { if (monitor.isComplete) { continue; } filename = monitor.filename; lineNumber = monitor.lineNumber; } gDebug.abort(filename, lineNumber); }.bind(this), function onSatisfied() { // The promise has been rejected, which means that we have satisfied // all completion conditions. }); promise = promise.then(function() { timeToCrash.reject(); }/* No error is possible here*/); } return promise; }, }); // List of well-known phases // Ideally, phases should be registered from the component that decides // when they start/stop. For compatibility with existing startup/shutdown // mechanisms, we register a few phases here. this.AsyncShutdown.profileChangeTeardown = getPhase("profile-change-teardown"); this.AsyncShutdown.profileBeforeChange = getPhase("profile-before-change"); this.AsyncShutdown.sendTelemetry = getPhase("profile-before-change2"); this.AsyncShutdown.webWorkersShutdown = getPhase("web-workers-shutdown"); this.AsyncShutdown.Barrier = Barrier; Object.freeze(this.AsyncShutdown);