Bug 1007723 - Telemetry (?) is causing a high memory consumption and lagging on browser startup. r=Yoric

toolkit/components/telemetry/TelemetryFile.jsm

@@ -31,6 +31,9 @@ const MAX_PING_FILE_AGE = 14 * 24 * 60 * 60 * 1000; // 2 weeks
 // MAX_PING_FILE_AGE indicate that we need to send all of our pings ASAP.
 const OVERDUE_PING_FILE_AGE = 7 * 24 * 60 * 60 * 1000; // 1 week
 
+// Maximum number of pings to save.
+const MAX_LRU_PINGS = 17;
+
 // The number of outstanding saved pings that we have issued loading
 // requests for.
 let pingsLoaded = 0;
@@ -58,6 +61,10 @@ this.TelemetryFile = {
     return OVERDUE_PING_FILE_AGE;
   },
 
+  get MAX_LRU_PINGS() {
+    return MAX_LRU_PINGS;
+  },
+
   get pingDirectoryPath() {
     return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
   },
@@ -142,12 +149,36 @@ this.TelemetryFile = {
 
       if (exists) {
         let entries = yield iter.nextBatch();
-        yield iter.close();
+        let sortedEntries = [];
 
-        let p = [e for (e of entries) if (!e.isDir)].
-            map((e) => this.loadHistograms(e.path));
+        for (let entry of entries) {
+          if (entry.isDir) {
+            continue;
+          }
 
-        yield Promise.all(p);
+          let info = yield OS.File.stat(entry.path);
+          sortedEntries.push({entry:entry, lastModificationDate: info.lastModificationDate});
+        }
+
+        sortedEntries.sort(function compare(a, b) {
+          return b.lastModificationDate - a.lastModificationDate;
+        });
+
+        let count = 0;
+        let result = [];
+
+        // Keep only the last MAX_LRU_PINGS entries to avoid that the backlog overgrows.
+        for (let i = 0; i < MAX_LRU_PINGS && i < sortedEntries.length; i++) {
+          let entry = sortedEntries[i].entry;
+          result.push(this.loadHistograms(entry.path))
+        }
+
+        for (let i = MAX_LRU_PINGS; i < sortedEntries.length; i++) {
+          let entry = sortedEntries[i].entry;
+          OS.File.remove(entry.path);
+        }
+
+        yield Promise.all(result);
       }
 
       yield iter.close();

toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js

@@ -37,6 +37,7 @@ const PING_TIMEOUT_LENGTH = 5000;
 const EXPIRED_PINGS = 5;
 const OVERDUE_PINGS = 6;
 const RECENT_PINGS = 4;
+const LRU_PINGS = TelemetryFile.MAX_LRU_PINGS;
 
 const TOTAL_EXPECTED_PINGS = OVERDUE_PINGS + RECENT_PINGS;
 
@@ -199,7 +200,7 @@ function run_test() {
  * Test that pings that are considered too old are just chucked out
  * immediately and never sent.
  */
-add_task(function test_expired_pings_are_deleted() {
+add_task(function* test_expired_pings_are_deleted() {
   let expiredPings = yield createSavedPings(EXPIRED_PINGS, EXPIRED_PING_FILE_AGE);
   yield startTelemetry();
   assertReceivedPings(0);
@@ -210,7 +211,7 @@ add_task(function test_expired_pings_are_deleted() {
 /**
  * Test that really recent pings are not sent on Telemetry initialization.
  */
-add_task(function test_recent_pings_not_sent() {
+add_task(function* test_recent_pings_not_sent() {
   let recentPings = yield createSavedPings(RECENT_PINGS);
   yield startTelemetry();
   assertReceivedPings(0);
@@ -218,12 +219,34 @@ add_task(function test_recent_pings_not_sent() {
   yield clearPings(recentPings);
 });
 
+/**
+ * Test that only the most recent LRU_PINGS pings are kept at startup.
+ */
+add_task(function* test_most_recent_pings_kept() {
+  let head = yield createSavedPings(LRU_PINGS);
+  let tail = yield createSavedPings(3, ONE_MINUTE_MS);
+  let pings = head.concat(tail);
+
+  yield startTelemetry();
+  let gen = TelemetryFile.popPendingPings();
+
+  for (let item of gen) {
+    for (let p of tail) {
+      do_check_neq(p.slug, item.slug);
+    }
+  }
+
+  assertNotSaved(tail);
+  yield resetTelemetry();
+  yield clearPings(pings);
+});
+
 /**
  * Create some recent, expired and overdue pings. The overdue pings should
  * trigger a send of all recent and overdue pings, but the expired pings
  * should just be deleted.
  */
-add_task(function test_overdue_pings_trigger_send() {
+add_task(function* test_overdue_pings_trigger_send() {
   let recentPings = yield createSavedPings(RECENT_PINGS);
   let expiredPings = yield createSavedPings(EXPIRED_PINGS, EXPIRED_PING_FILE_AGE);
   let overduePings = yield createSavedPings(OVERDUE_PINGS, OVERDUE_PING_FILE_AGE);
@@ -237,6 +260,6 @@ add_task(function test_overdue_pings_trigger_send() {
   yield resetTelemetry();
 });
 
-add_task(function teardown() {
+add_task(function* teardown() {
   yield stopHttpServer();
 });