mirror of
https://gitlab.winehq.org/wine/wine-gecko.git
synced 2024-09-13 09:24:08 -07:00
Bug 1186955 - Telemetry should discard pings that are too big. r=gfritzsche
This commit is contained in:
parent
a07853ed6d
commit
bad1a8bb5e
@ -4707,6 +4707,48 @@
|
|||||||
"n_buckets": 20,
|
"n_buckets": 20,
|
||||||
"description": "Time (ms) it takes for checking if the pending pings are over-quota"
|
"description": "Time (ms) it takes for checking if the pending pings are over-quota"
|
||||||
},
|
},
|
||||||
|
"TELEMETRY_PING_SIZE_EXCEEDED_SEND": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "count",
|
||||||
|
"description": "Number of Telemetry pings discarded before sending because they exceeded the maximum size"
|
||||||
|
},
|
||||||
|
"TELEMETRY_PING_SIZE_EXCEEDED_PENDING": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "count",
|
||||||
|
"description": "Number of Telemetry pending pings discarded because they exceeded the maximum size"
|
||||||
|
},
|
||||||
|
"TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "count",
|
||||||
|
"description": "Number of archived Telemetry pings discarded because they exceeded the maximum size"
|
||||||
|
},
|
||||||
|
"TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "linear",
|
||||||
|
"high": "30",
|
||||||
|
"n_buckets": 29,
|
||||||
|
"description": "The size (MB) of the Telemetry pending pings exceeding the maximum file size"
|
||||||
|
},
|
||||||
|
"TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "linear",
|
||||||
|
"high": "30",
|
||||||
|
"n_buckets": 29,
|
||||||
|
"description": "The size (MB) of the Telemetry archived, compressed, pings exceeding the maximum file size"
|
||||||
|
},
|
||||||
|
"TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB": {
|
||||||
|
"alert_emails": ["telemetry-client-dev@mozilla.com"],
|
||||||
|
"expires_in_version": "never",
|
||||||
|
"kind": "linear",
|
||||||
|
"high": "30",
|
||||||
|
"n_buckets": 29,
|
||||||
|
"description": "The size (MB) of the ping data submitted to Telemetry exceeding the maximum size"
|
||||||
|
},
|
||||||
"TELEMETRY_DISCARDED_CONTENT_PINGS_COUNT": {
|
"TELEMETRY_DISCARDED_CONTENT_PINGS_COUNT": {
|
||||||
"alert_emails": ["perf-telemetry-alerts@mozilla.com"],
|
"alert_emails": ["perf-telemetry-alerts@mozilla.com"],
|
||||||
"expires_in_version": "never",
|
"expires_in_version": "never",
|
||||||
|
@ -928,6 +928,19 @@ let TelemetrySendImpl = {
|
|||||||
let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(networkPayload));
|
let utf8Payload = converter.ConvertFromUnicode(JSON.stringify(networkPayload));
|
||||||
utf8Payload += converter.Finish();
|
utf8Payload += converter.Finish();
|
||||||
Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(new Date() - startTime);
|
Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(new Date() - startTime);
|
||||||
|
|
||||||
|
// Check the size and drop pings which are too big.
|
||||||
|
const pingSizeBytes = utf8Payload.length;
|
||||||
|
if (pingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
|
||||||
|
this._log.error("_doPing - submitted ping exceeds the size limit, size: " + pingSizeBytes);
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add();
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
|
||||||
|
.add(Math.floor(pingSizeBytes / 1024 / 1024));
|
||||||
|
// We don't need to call |request.abort()| as it was not sent yet.
|
||||||
|
this._pendingPingRequests.delete(id);
|
||||||
|
return TelemetryStorage.removePendingPing(id);
|
||||||
|
}
|
||||||
|
|
||||||
let payloadStream = Cc["@mozilla.org/io/string-input-stream;1"]
|
let payloadStream = Cc["@mozilla.org/io/string-input-stream;1"]
|
||||||
.createInstance(Ci.nsIStringInputStream);
|
.createInstance(Ci.nsIStringInputStream);
|
||||||
startTime = new Date();
|
startTime = new Date();
|
||||||
|
@ -59,6 +59,9 @@ const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB
|
|||||||
// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).
|
// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).
|
||||||
const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB
|
const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB
|
||||||
|
|
||||||
|
// The maximum size a pending/archived ping can take on disk.
|
||||||
|
const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB
|
||||||
|
|
||||||
// This special value is submitted when the archive is outside of the quota.
|
// This special value is submitted when the archive is outside of the quota.
|
||||||
const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;
|
const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;
|
||||||
|
|
||||||
@ -135,6 +138,12 @@ this.TelemetryStorage = {
|
|||||||
return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
|
return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The maximum size a ping can have, in bytes.
|
||||||
|
*/
|
||||||
|
get MAXIMUM_PING_SIZE() {
|
||||||
|
return PING_FILE_MAXIMUM_SIZE_BYTES;
|
||||||
|
},
|
||||||
/**
|
/**
|
||||||
* Shutdown & block on any outstanding async activity in this module.
|
* Shutdown & block on any outstanding async activity in this module.
|
||||||
*
|
*
|
||||||
@ -644,13 +653,27 @@ let TelemetryStorageImpl = {
|
|||||||
const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);
|
const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);
|
||||||
const pathCompressed = path + "lz4";
|
const pathCompressed = path + "lz4";
|
||||||
|
|
||||||
|
// Purge pings which are too big.
|
||||||
|
let checkSize = function*(path) {
|
||||||
|
const fileSize = (yield OS.File.stat(path)).size;
|
||||||
|
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")
|
||||||
|
.add(Math.floor(fileSize / 1024 / 1024));
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();
|
||||||
|
yield OS.File.remove(path, {ignoreAbsent: true});
|
||||||
|
throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Try to load a compressed version of the archived ping first.
|
// Try to load a compressed version of the archived ping first.
|
||||||
this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed);
|
this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed);
|
||||||
|
yield* checkSize(pathCompressed);
|
||||||
return yield this.loadPingFile(pathCompressed, /*compressed*/ true);
|
return yield this.loadPingFile(pathCompressed, /*compressed*/ true);
|
||||||
} catch (ex if ex.becauseNoSuchFile) {
|
} catch (ex if ex.becauseNoSuchFile) {
|
||||||
// If that fails, look for the uncompressed version.
|
// If that fails, look for the uncompressed version.
|
||||||
this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path);
|
this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path);
|
||||||
|
yield* checkSize(path);
|
||||||
return yield this.loadPingFile(path, /*compressed*/ false);
|
return yield this.loadPingFile(path, /*compressed*/ false);
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
@ -671,6 +694,8 @@ let TelemetryStorageImpl = {
|
|||||||
this._log.trace("_removeArchivedPing - removing ping from: " + path);
|
this._log.trace("_removeArchivedPing - removing ping from: " + path);
|
||||||
yield OS.File.remove(path, {ignoreAbsent: true});
|
yield OS.File.remove(path, {ignoreAbsent: true});
|
||||||
yield OS.File.remove(pathCompressed, {ignoreAbsent: true});
|
yield OS.File.remove(pathCompressed, {ignoreAbsent: true});
|
||||||
|
// Remove the ping from the cache.
|
||||||
|
this._archivedPings.delete(id);
|
||||||
}),
|
}),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -813,6 +838,19 @@ let TelemetryStorageImpl = {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Enforce a maximum file size limit on archived pings.
|
||||||
|
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||||
|
this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize);
|
||||||
|
// We just remove the ping from the disk, we don't bother removing it from pingList
|
||||||
|
// since it won't contribute to the quota.
|
||||||
|
yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type)
|
||||||
|
.catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id));
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB")
|
||||||
|
.add(Math.floor(fileSize / 1024 / 1024));
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
archiveSizeInBytes += fileSize;
|
archiveSizeInBytes += fileSize;
|
||||||
|
|
||||||
if (archiveSizeInBytes < SAFE_QUOTA) {
|
if (archiveSizeInBytes < SAFE_QUOTA) {
|
||||||
@ -857,9 +895,6 @@ let TelemetryStorageImpl = {
|
|||||||
// This list is guaranteed to be in order, so remove the pings at its
|
// This list is guaranteed to be in order, so remove the pings at its
|
||||||
// beginning (oldest).
|
// beginning (oldest).
|
||||||
yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);
|
yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);
|
||||||
|
|
||||||
// Remove outdated pings from the cache.
|
|
||||||
this._archivedPings.delete(ping.id);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const endTimeStamp = Policy.now().getTime();
|
const endTimeStamp = Policy.now().getTime();
|
||||||
@ -1191,15 +1226,36 @@ let TelemetryStorageImpl = {
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
loadPendingPing: function(id) {
|
loadPendingPing: Task.async(function*(id) {
|
||||||
this._log.trace("loadPendingPing - id: " + id);
|
this._log.trace("loadPendingPing - id: " + id);
|
||||||
let info = this._pendingPings.get(id);
|
let info = this._pendingPings.get(id);
|
||||||
if (!info) {
|
if (!info) {
|
||||||
this._log.trace("loadPendingPing - unknown id " + id);
|
this._log.trace("loadPendingPing - unknown id " + id);
|
||||||
return Promise.reject(new Error("TelemetryStorage.loadPendingPing - no ping with id " + id));
|
throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id);
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.loadPingFile(info.path, false).catch(e => {
|
// Try to get the dimension of the ping. If that fails, update the histograms.
|
||||||
|
let fileSize = 0;
|
||||||
|
try {
|
||||||
|
fileSize = (yield OS.File.stat(info.path)).size;
|
||||||
|
} catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) {
|
||||||
|
// Fall through and let |loadPingFile| report the error.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Purge pings which are too big.
|
||||||
|
if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||||
|
yield this.removePendingPing(id);
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")
|
||||||
|
.add(Math.floor(fileSize / 1024 / 1024));
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();
|
||||||
|
throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to load the ping file. Update the related histograms on failure.
|
||||||
|
let ping;
|
||||||
|
try {
|
||||||
|
ping = yield this.loadPingFile(info.path, false);
|
||||||
|
} catch(e) {
|
||||||
// If we failed to load the ping, check what happened and update the histogram.
|
// If we failed to load the ping, check what happened and update the histogram.
|
||||||
// Then propagate the rejection.
|
// Then propagate the rejection.
|
||||||
if (e instanceof PingReadError) {
|
if (e instanceof PingReadError) {
|
||||||
@ -1207,10 +1263,11 @@ let TelemetryStorageImpl = {
|
|||||||
} else if (e instanceof PingParseError) {
|
} else if (e instanceof PingParseError) {
|
||||||
Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add();
|
Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add();
|
||||||
}
|
}
|
||||||
|
throw e;
|
||||||
|
};
|
||||||
|
|
||||||
return Promise.reject(e);
|
return ping;
|
||||||
});
|
}),
|
||||||
},
|
|
||||||
|
|
||||||
removePendingPing: function(id) {
|
removePendingPing: function(id) {
|
||||||
let info = this._pendingPings.get(id);
|
let info = this._pendingPings.get(id);
|
||||||
@ -1281,6 +1338,21 @@ let TelemetryStorageImpl = {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Enforce a maximum file size limit on pending pings.
|
||||||
|
if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {
|
||||||
|
this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path);
|
||||||
|
try {
|
||||||
|
yield OS.File.remove(file.path);
|
||||||
|
} catch (ex) {
|
||||||
|
this._log.error("_scanPendingPings - failed to remove file " + file.path, ex);
|
||||||
|
} finally {
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB")
|
||||||
|
.add(Math.floor(info.size / 1024 / 1024));
|
||||||
|
Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let id = OS.Path.basename(file.path);
|
let id = OS.Path.basename(file.path);
|
||||||
if (!UUID_REGEX.test(id)) {
|
if (!UUID_REGEX.test(id)) {
|
||||||
this._log.trace("_scanPendingPings - filename is not a UUID: " + id);
|
this._log.trace("_scanPendingPings - filename is not a UUID: " + id);
|
||||||
|
Loading…
Reference in New Issue
Block a user