diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index 91db5a15a55..a5a2319a780 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -4707,6 +4707,48 @@ "n_buckets": 20, "description": "Time (ms) it takes for checking if the pending pings are over-quota" }, + "TELEMETRY_PING_SIZE_EXCEEDED_SEND": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "count", + "description": "Number of Telemetry pings discarded before sending because they exceeded the maximum size" + }, + "TELEMETRY_PING_SIZE_EXCEEDED_PENDING": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "count", + "description": "Number of Telemetry pending pings discarded because they exceeded the maximum size" + }, + "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "count", + "description": "Number of archived Telemetry pings discarded because they exceeded the maximum size" + }, + "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "linear", + "high": "30", + "n_buckets": 29, + "description": "The size (MB) of the Telemetry pending pings exceeding the maximum file size" + }, + "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "linear", + "high": "30", + "n_buckets": 29, + "description": "The size (MB) of the Telemetry archived, compressed, pings exceeding the maximum file size" + }, + "TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB": { + "alert_emails": ["telemetry-client-dev@mozilla.com"], + "expires_in_version": "never", + "kind": "linear", + "high": "30", + "n_buckets": 29, + "description": "The size (MB) of the ping data submitted to Telemetry exceeding the maximum size" + }, "TELEMETRY_DISCARDED_CONTENT_PINGS_COUNT": { "alert_emails": ["perf-telemetry-alerts@mozilla.com"], "expires_in_version": "never", diff --git a/toolkit/components/telemetry/TelemetrySend.jsm b/toolkit/components/telemetry/TelemetrySend.jsm index bc7f918b267..449357798bf 100644 --- a/toolkit/components/telemetry/TelemetrySend.jsm +++ b/toolkit/components/telemetry/TelemetrySend.jsm @@ -928,6 +928,19 @@ let TelemetrySendImpl = { let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(networkPayload)); utf8Payload += converter.Finish(); Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(new Date() - startTime); + + // Check the size and drop pings which are too big. + const pingSizeBytes = utf8Payload.length; + if (pingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) { + this._log.error("_doPing - submitted ping exceeds the size limit, size: " + pingSizeBytes); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add(); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB") + .add(Math.floor(pingSizeBytes / 1024 / 1024)); + // We don't need to call |request.abort()| as it was not sent yet. + this._pendingPingRequests.delete(id); + return TelemetryStorage.removePendingPing(id); + } + let payloadStream = Cc["@mozilla.org/io/string-input-stream;1"] .createInstance(Ci.nsIStringInputStream); startTime = new Date(); diff --git a/toolkit/components/telemetry/TelemetryStorage.jsm b/toolkit/components/telemetry/TelemetryStorage.jsm index 14fa60d39f0..c303e7e762a 100644 --- a/toolkit/components/telemetry/TelemetryStorage.jsm +++ b/toolkit/components/telemetry/TelemetryStorage.jsm @@ -59,6 +59,9 @@ const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB // Maximum space the outgoing pings can take on disk, for Mobile (in Bytes). const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB +// The maximum size a pending/archived ping can take on disk. +const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB + // This special value is submitted when the archive is outside of the quota. const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300; @@ -135,6 +138,12 @@ this.TelemetryStorage = { return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings"); }, + /** + * The maximum size a ping can have, in bytes. + */ + get MAXIMUM_PING_SIZE() { + return PING_FILE_MAXIMUM_SIZE_BYTES; + }, /** * Shutdown & block on any outstanding async activity in this module. * @@ -644,13 +653,27 @@ let TelemetryStorageImpl = { const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type); const pathCompressed = path + "lz4"; + // Purge pings which are too big. + let checkSize = function*(path) { + const fileSize = (yield OS.File.stat(path)).size; + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add(); + yield OS.File.remove(path, {ignoreAbsent: true}); + throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize); + } + }; + try { // Try to load a compressed version of the archived ping first. this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed); + yield* checkSize(pathCompressed); return yield this.loadPingFile(pathCompressed, /*compressed*/ true); } catch (ex if ex.becauseNoSuchFile) { // If that fails, look for the uncompressed version. this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path); + yield* checkSize(path); return yield this.loadPingFile(path, /*compressed*/ false); } }), @@ -671,6 +694,8 @@ let TelemetryStorageImpl = { this._log.trace("_removeArchivedPing - removing ping from: " + path); yield OS.File.remove(path, {ignoreAbsent: true}); yield OS.File.remove(pathCompressed, {ignoreAbsent: true}); + // Remove the ping from the cache. + this._archivedPings.delete(id); }), /** @@ -813,6 +838,19 @@ let TelemetryStorageImpl = { continue; } + // Enforce a maximum file size limit on archived pings. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize); + // We just remove the ping from the disk, we don't bother removing it from pingList + // since it won't contribute to the quota. + yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type) + .catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id)); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add(); + continue; + } + archiveSizeInBytes += fileSize; if (archiveSizeInBytes < SAFE_QUOTA) { @@ -857,9 +895,6 @@ let TelemetryStorageImpl = { // This list is guaranteed to be in order, so remove the pings at its // beginning (oldest). yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type); - - // Remove outdated pings from the cache. - this._archivedPings.delete(ping.id); } const endTimeStamp = Policy.now().getTime(); @@ -1191,15 +1226,36 @@ let TelemetryStorageImpl = { }); }, - loadPendingPing: function(id) { + loadPendingPing: Task.async(function*(id) { this._log.trace("loadPendingPing - id: " + id); let info = this._pendingPings.get(id); if (!info) { this._log.trace("loadPendingPing - unknown id " + id); - return Promise.reject(new Error("TelemetryStorage.loadPendingPing - no ping with id " + id)); + throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id); } - return this.loadPingFile(info.path, false).catch(e => { + // Try to get the dimension of the ping. If that fails, update the histograms. + let fileSize = 0; + try { + fileSize = (yield OS.File.stat(info.path)).size; + } catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) { + // Fall through and let |loadPingFile| report the error. + } + + // Purge pings which are too big. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + yield this.removePendingPing(id); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add(); + throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize); + } + + // Try to load the ping file. Update the related histograms on failure. + let ping; + try { + ping = yield this.loadPingFile(info.path, false); + } catch(e) { // If we failed to load the ping, check what happened and update the histogram. // Then propagate the rejection. if (e instanceof PingReadError) { @@ -1207,10 +1263,11 @@ let TelemetryStorageImpl = { } else if (e instanceof PingParseError) { Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add(); } + throw e; + }; - return Promise.reject(e); - }); - }, + return ping; + }), removePendingPing: function(id) { let info = this._pendingPings.get(id); @@ -1281,6 +1338,21 @@ let TelemetryStorageImpl = { continue; } + // Enforce a maximum file size limit on pending pings. + if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path); + try { + yield OS.File.remove(file.path); + } catch (ex) { + this._log.error("_scanPendingPings - failed to remove file " + file.path, ex); + } finally { + Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB") + .add(Math.floor(info.size / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add(); + continue; + } + } + let id = OS.Path.basename(file.path); if (!UUID_REGEX.test(id)) { this._log.trace("_scanPendingPings - filename is not a UUID: " + id);