Bug 777328 - Gather telemetry data for how much cache corruption reduction plan would help. r=michal

netwerk/cache/nsCacheService.cpp

@@ -2852,6 +2852,12 @@ nsCacheService::ClearPendingRequests(nsCacheEntry * entry)
     }
 }
 
+bool
+nsCacheService::IsDoomListEmpty()
+{
+    nsCacheEntry * entry = (nsCacheEntry *)PR_LIST_HEAD(&mDoomedEntries);
+    return &mDoomedEntries == entry;
+}
 
 void
 nsCacheService::ClearDoomList()

netwerk/cache/nsCacheService.h

@@ -178,6 +178,7 @@ public:
     { gService->mLock.AssertCurrentThreadOwns(); }
 
     static void      LeavePrivateBrowsing();
+    bool             IsDoomListEmpty();
 
     typedef bool (*DoomCheckFn)(nsCacheEntry* entry);
 
@@ -190,6 +191,7 @@ private:
     friend class nsSetDiskSmartSizeCallback;
     friend class nsDoomEvent;
     friend class nsDisableOldMaxSmartSizePrefEvent;
+    friend class nsDiskCacheMap;
 
     /**
      * Internal Methods

netwerk/cache/nsDiskCache.h

@@ -49,7 +49,10 @@ public:
       kBlockFileSizeLessThanBitMap = 19,
       kBlockFileBitMapReadError = 20,
       kBlockFileEstimatedSizeError = 21,
-      kFlushHeaderError = 22
+      kFlushHeaderError = 22,
+      kCacheCleanFilePathError = 23,
+      kCacheCleanOpenFileError = 24,
+      kCacheCleanTimerError = 25
     };
 
     // Parameter initval initializes internal state of hash function. Hash values are different

netwerk/cache/nsDiskCacheMap.cpp

@@ -7,6 +7,7 @@
 #include "nsDiskCacheMap.h"
 #include "nsDiskCacheBinding.h"
 #include "nsDiskCacheEntry.h"
+#include "nsCacheService.h"
 
 #include "nsCache.h"
 
@@ -18,6 +19,8 @@
 
 #include "mozilla/Telemetry.h"
 
+using namespace mozilla;
+
 /******************************************************************************
  *  nsDiskCacheMap
  *****************************************************************************/
@@ -56,9 +59,14 @@ nsDiskCacheMap::Open(nsIFile *  cacheDirectory,
 
     bool cacheFilesExist = CacheFilesExist();
     rv = NS_ERROR_FILE_CORRUPTED;  // presume the worst
+    PRUint32 mapSize = PR_Available(mMapFD);    
+
+    if (NS_FAILED(InitCacheClean(cacheDirectory, corruptInfo))) {
+        // corruptInfo is set in the call to InitCacheClean
+        goto error_exit;
+    }
 
     // check size of map file
-    PRUint32 mapSize = PR_Available(mMapFD);    
     if (mapSize == 0) {  // creating a new _CACHE_MAP_
 
         // block files shouldn't exist if we're creating the _CACHE_MAP_
@@ -173,8 +181,7 @@ nsDiskCacheMap::Open(nsIFile *  cacheDirectory,
         // extra scope so the compiler doesn't barf on the above gotos jumping
         // past this declaration down here
         PRUint32 overhead = moz_malloc_size_of(mRecordArray);
-        mozilla::Telemetry::Accumulate(mozilla::Telemetry::HTTP_DISK_CACHE_OVERHEAD,
-                overhead);
+        Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead);
     }
 
     *corruptInfo = nsDiskCache::kNotCorrupt;
@@ -192,6 +199,12 @@ nsDiskCacheMap::Close(bool flush)
 {
     nsresult  rv = NS_OK;
 
+    // Cancel any pending cache validation event, the FlushRecords call below
+    // will validate the cache.
+    if (mCleanCacheTimer) {
+        mCleanCacheTimer->Cancel();
+    }
+
     // If cache map file and its block files are still open, close them
     if (mMapFD) {
         // close block files
@@ -210,6 +223,12 @@ nsDiskCacheMap::Close(bool flush)
 
         mMapFD = nullptr;
     }
+
+    if (mCleanFD) {
+        PR_Close(mCleanFD);
+        mCleanFD = nullptr;
+    }
+
     PR_FREEIF(mRecordArray);
     PR_FREEIF(mBuffer);
     mBufferSize = 0;
@@ -235,6 +254,7 @@ nsDiskCacheMap::Trim()
 nsresult
 nsDiskCacheMap::FlushHeader()
 {
+    RevalidateCache();
     if (!mMapFD)  return NS_ERROR_NOT_AVAILABLE;
     
     // seek to beginning of cache map
@@ -347,6 +367,9 @@ nsDiskCacheMap::GrowRecords()
     // Set as the new record array
     mRecordArray = newArray;
     mHeader.mRecordCount = newCount;
+
+    InvalidateCache();
+
     return NS_OK;
 }
 
@@ -393,6 +416,9 @@ nsDiskCacheMap::ShrinkRecords()
     // Set as the new record array
     mRecordArray = newArray;
     mHeader.mRecordCount = newCount;
+
+    InvalidateCache();
+
     return NS_OK;
 }
 
@@ -421,6 +447,7 @@ nsDiskCacheMap::AddRecord( nsDiskCacheRecord *  mapRecord,
         mHeader.mBucketUsage[bucketIndex]++;           
         if (mHeader.mEvictionRank[bucketIndex] < mapRecord->EvictionRank())
             mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank();
+        InvalidateCache();
     } else {
         // Find the record with the highest eviction rank
         nsDiskCacheRecord * mostEvictable = &records[0];
@@ -436,6 +463,7 @@ nsDiskCacheMap::AddRecord( nsDiskCacheRecord *  mapRecord,
             mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank();
         if (oldRecord->EvictionRank() >= mHeader.mEvictionRank[bucketIndex]) 
             mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0);
+        InvalidateCache();
     }
 
     NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0),
@@ -466,6 +494,8 @@ nsDiskCacheMap::UpdateRecord( nsDiskCacheRecord *  mapRecord)
             else if (mHeader.mEvictionRank[bucketIndex] == oldRank)
                 mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0);
 
+            InvalidateCache();
+
 NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0),
              "eviction rank out of sync");
             return NS_OK;
@@ -521,6 +551,8 @@ nsDiskCacheMap::DeleteRecord( nsDiskCacheRecord *  mapRecord)
                 mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0);
             }
 
+            InvalidateCache();
+
             NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] ==
                          GetBucketRank(bucketIndex, 0), "eviction rank out of sync");
             return NS_OK;
@@ -551,6 +583,7 @@ nsDiskCacheMap::VisitEachRecord(PRUint32                    bucketIndex,
             --count;
             records[i] = records[count];
             records[count].SetHashNumber(0);
+            InvalidateCache();
         }
     }
 
@@ -1174,3 +1207,186 @@ nsDiskCacheMap::NotifyCapacityChange(PRUint32 capacity)
     mMaxRecordCount = maxRecordCount;
   }
 }
+
+nsresult
+nsDiskCacheMap::InitCacheClean(nsIFile *  cacheDirectory,
+                               nsDiskCache::CorruptCacheInfo *  corruptInfo)
+{
+    // The _CACHE_CLEAN_ file will be used in the future to determine
+    // if the cache is clean or not. 
+    bool cacheCleanFileExists = false;
+    nsCOMPtr<nsIFile> cacheCleanFile;
+    nsresult rv = cacheDirectory->Clone(getter_AddRefs(cacheCleanFile));
+    if (NS_SUCCEEDED(rv)) {
+        rv = cacheCleanFile->AppendNative(
+                 NS_LITERAL_CSTRING("_CACHE_CLEAN_"));
+        if (NS_SUCCEEDED(rv)) {
+            // Check if the file already exists, if it does, we will later read the
+            // value and report it to telemetry.
+            cacheCleanFile->Exists(&cacheCleanFileExists);
+        }
+    }
+    if (NS_FAILED(rv)) {
+        NS_WARNING("Could not build cache clean file path");
+        *corruptInfo = nsDiskCache::kCacheCleanFilePathError;
+        return rv;
+    }
+
+    // Make sure the _CACHE_CLEAN_ file exists
+    rv = cacheCleanFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE,
+                                          00600, &mCleanFD);
+    if (NS_FAILED(rv)) {
+        NS_WARNING("Could not open cache clean file");
+        *corruptInfo = nsDiskCache::kCacheCleanOpenFileError;
+        return rv;
+    }
+
+    if (cacheCleanFileExists) {
+        char clean = '0';
+        PRInt32 bytesRead = PR_Read(mCleanFD, &clean, 1);
+        if (bytesRead != 1) {
+            NS_WARNING("Could not read _CACHE_CLEAN_ file contents");
+        } else {
+            Telemetry::Accumulate(Telemetry::DISK_CACHE_REDUCTION_TRIAL,
+                                  clean == '1' ? 1 : 0);
+        }
+    }
+
+    // Create a timer that will be used to validate the cache
+    // as long as an activity threshold was met
+    mCleanCacheTimer = do_CreateInstance("@mozilla.org/timer;1", &rv);
+    if (NS_SUCCEEDED(rv)) {
+        mCleanCacheTimer->SetTarget(nsCacheService::GlobalInstance()->mCacheIOThread);
+        rv = ResetCacheTimer();
+    }
+
+    if (NS_FAILED(rv)) {
+        NS_WARNING("Could not create cache clean timer");
+        mCleanCacheTimer = nullptr;
+        *corruptInfo = nsDiskCache::kCacheCleanTimerError;
+        return rv;
+    }
+
+    return NS_OK;
+}
+
+nsresult
+nsDiskCacheMap::WriteCacheClean(bool clean)
+{
+    nsCacheService::AssertOwnsLock();
+    CACHE_LOG_DEBUG(("CACHE: WriteCacheClean: %d\n", clean? 1 : 0));
+    // I'm using a simple '1' or '0' to denote cache clean
+    // since it can be edited easily by any text editor for testing.
+    char data = clean? '1' : '0';
+    PRInt32 filePos = PR_Seek(mCleanFD, 0, PR_SEEK_SET);
+    if (filePos != 0) {
+        NS_WARNING("Could not seek in cache map file!");
+        return NS_ERROR_FAILURE;
+    }
+    PRInt32 bytesWritten = PR_Write(mCleanFD, &data, 1);
+    if (bytesWritten != 1) {
+        NS_WARNING("Could not write cache map file!");
+        return NS_ERROR_FAILURE;
+    }
+    PRStatus err = PR_Sync(mCleanFD);
+    if (err != PR_SUCCESS) {
+        NS_WARNING("Could not flush mCleanFD!");
+    }
+
+    return NS_OK;
+}
+
+nsresult
+nsDiskCacheMap::InvalidateCache()
+{
+    nsCacheService::AssertOwnsLock();
+    CACHE_LOG_DEBUG(("CACHE: InvalidateCache\n"));
+    nsresult rv;
+  
+    if (!mIsDirtyCacheFlushed) {
+        rv = WriteCacheClean(false);
+        NS_ENSURE_SUCCESS(rv, rv);
+        mIsDirtyCacheFlushed = true;
+    }
+
+    rv = ResetCacheTimer();
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    return NS_OK;
+}
+
+nsresult
+nsDiskCacheMap::ResetCacheTimer(PRInt32 timeout)
+{
+    mCleanCacheTimer->Cancel();
+    nsresult rv =
+      mCleanCacheTimer->InitWithFuncCallback(RevalidateTimerCallback,
+                                             this, timeout,
+                                             nsITimer::TYPE_ONE_SHOT);
+    NS_ENSURE_SUCCESS(rv, rv);
+    mLastInvalidateTime = PR_IntervalNow();
+
+    return rv;
+}
+
+void
+nsDiskCacheMap::RevalidateTimerCallback(nsITimer *aTimer, void *arg)
+{
+    nsDiskCacheMap *diskCacheMap = reinterpret_cast<nsDiskCacheMap *>(arg);
+    nsresult rv;
+
+    // Intentional braces to scope mutex to only what is needed
+    {
+        nsCacheServiceAutoLock lock(LOCK_TELEM(NSDISKCACHEMAP_REVALIDATION));
+        // If we have less than kLastInvalidateTime since the last timer was
+        // issued then another thread called InvalidateCache.  This won't catch
+        // all cases where we wanted to cancel the timer, but under the lock it
+        // is always OK to revalidate as long as IsCacheInSafeState() returns
+        // true.  We just want to avoid revalidating when we can to reduce IO
+        // and this check will do that.
+        PRUint32 delta =
+            PR_IntervalToMilliseconds(PR_IntervalNow() -
+                                      diskCacheMap->mLastInvalidateTime) +
+            kRevalidateCacheTimeoutTolerance;
+        if (delta < kRevalidateCacheTimeout) {
+            diskCacheMap->ResetCacheTimer();
+            return;
+        }
+        rv = diskCacheMap->RevalidateCache();
+    }
+
+    if (NS_FAILED(rv)) {
+        diskCacheMap->ResetCacheTimer(kRevalidateCacheErrorTimeout);
+    }
+}
+
+bool
+nsDiskCacheMap::IsCacheInSafeState()
+{
+    return nsCacheService::GlobalInstance()->IsDoomListEmpty();
+}
+
+nsresult
+nsDiskCacheMap::RevalidateCache()
+{
+    CACHE_LOG_DEBUG(("CACHE: RevalidateCache\n"));
+    nsresult rv;
+
+    if (!IsCacheInSafeState()) {
+        CACHE_LOG_DEBUG(("CACHE: Revalidation not performed because "
+                         "cache not in a safe state\n"));
+        return NS_ERROR_FAILURE;
+    }
+
+    // We want this after the lock to prove that flushing a file isn't that expensive
+    Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_REVALIDATION> totalTimer;
+
+    // If telemetry data shows it is worth it, we'll be flushing headers and
+    // records before flushing the clean cache file.
+  
+    // Write out the _CACHE_CLEAN_ file with '1'
+    rv = WriteCacheClean(true);
+    mIsDirtyCacheFlushed = false;
+
+    return NS_OK;
+}

netwerk/cache/nsDiskCacheMap.h

@@ -14,6 +14,7 @@
 #include "nsDebug.h"
 #include "nsError.h"
 #include "nsIFile.h"
+#include "nsITimer.h"
 
 #include "nsDiskCache.h"
 #include "nsDiskCacheBlockFile.h"
@@ -78,6 +79,12 @@ struct nsDiskCacheEntry;
 // preallocate up to 1MB of separate cache file
 #define kPreallocateLimit  1 * 1024 * 1024
 
+// The minimum amount of milliseconds to wait before re-attempting to
+// revalidate the cache.
+#define kRevalidateCacheTimeout 5000
+#define kRevalidateCacheTimeoutTolerance 10
+#define kRevalidateCacheErrorTimeout 1000
+
 class nsDiskCacheRecord {
 
 private:
@@ -378,13 +385,17 @@ public:
      nsDiskCacheMap() : 
         mCacheDirectory(nullptr),
         mMapFD(nullptr),
+        mCleanFD(nullptr),
         mRecordArray(nullptr),
         mBufferSize(0),
         mBuffer(nullptr),
-        mMaxRecordCount(16384) // this default value won't matter
+        mMaxRecordCount(16384), // this default value won't matter
+        mIsDirtyCacheFlushed(false),
+        mLastInvalidateTime(0)
     { }
 
-    ~nsDiskCacheMap() {
+    ~nsDiskCacheMap()
+    {
         (void) Close(true);
     }
 
@@ -526,18 +537,39 @@ private:
     nsDiskCacheEntry *  CreateDiskCacheEntry(nsDiskCacheBinding *  binding,
                                              PRUint32 * size);
 
-/**
+    // Initializes the _CACHE_CLEAN_ related functionality
+    nsresult InitCacheClean(nsIFile *  cacheDirectory,
+                            nsDiskCache::CorruptCacheInfo *  corruptInfo);
+    // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file
+    nsresult WriteCacheClean(bool clean);
+    // Resets the timout for revalidating the cache
+    nsresult ResetCacheTimer(PRInt32 timeout = kRevalidateCacheTimeout);
+    // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer
+    nsresult InvalidateCache();
+    // Determines if the cache is in a safe state
+    bool IsCacheInSafeState();
+    // Revalidates the cache by writting out the header, records, and finally
+    // by calling WriteCacheClean(true).
+    nsresult RevalidateCache();
+    // Timer which revalidates the cache
+    static void RevalidateTimerCallback(nsITimer *aTimer, void *arg);
+
+/** 
  *  data members
  */
 private:
+    nsCOMPtr<nsITimer>      mCleanCacheTimer;
     nsCOMPtr<nsIFile>       mCacheDirectory;
     PRFileDesc *            mMapFD;
+    PRFileDesc *            mCleanFD;
     nsDiskCacheRecord *     mRecordArray;
     nsDiskCacheBlockFile    mBlockFile[kNumBlockFiles];
     PRUint32                mBufferSize;
     char *                  mBuffer;
     nsDiskCacheHeader       mHeader;
     PRInt32                 mMaxRecordCount;
+    bool                    mIsDirtyCacheFlushed;
+    PRIntervalTime          mLastInvalidateTime;
 };
 
 #endif // _nsDiskCacheMap_h_

toolkit/components/telemetry/TelemetryHistograms.h

@@ -201,6 +201,7 @@ HISTOGRAM(SPDY_SETTINGS_IW, 1, 1000, 50, EXPONENTIAL,  "SPDY: Settings IW (round
 #undef HTTP_HISTOGRAMS
 
 HISTOGRAM(DISK_CACHE_CORRUPT_DETAILS, 1, 50, 51, LINEAR, "Why the HTTP disk cache was corrupted at startup")
+HISTOGRAM_BOOLEAN(DISK_CACHE_REDUCTION_TRIAL, "Stores 1 if the cache would be corrupted with the disk cache corruption plan of Bug 105843")
 HISTOGRAM_ENUMERATED_VALUES(HTTP_CACHE_DISPOSITION_2,         5, "HTTP Cache Hit, Reval, Failed-Reval, Miss")
 HISTOGRAM_ENUMERATED_VALUES(HTTP_DISK_CACHE_DISPOSITION_2,    5, "HTTP Disk Cache Hit, Reval, Failed-Reval, Miss")
 HISTOGRAM_ENUMERATED_VALUES(HTTP_MEMORY_CACHE_DISPOSITION_2,  5, "HTTP Memory Cache Hit, Reval, Failed-Reval, Miss")
@@ -279,6 +280,7 @@ CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETPREDICTEDDATASIZE)
 CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETLASTFETCHED)
 CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETCLIENTID)
 CACHE_LOCK_HISTOGRAM(NSBLOCKONCACHETHREADEVENT_RUN)
+CACHE_LOCK_HISTOGRAM(NSDISKCACHEMAP_REVALIDATION)
 
 #undef CACHE_LOCK_HISTOGRAM
  
@@ -332,6 +334,7 @@ HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR, 1, 10000, 10, EXPONENTIAL, "Time spent d
 HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Time spent during showdown stopping thread deleting old disk cache (ms)")
 HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Total Time spent (ms) during disk cache showdown")
 HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN_CLEAR_PRIVATE, 1, 10000, 10, EXPONENTIAL, "Time spent (ms) during showdown deleting disk cache for 'clear private data' option")
+HISTOGRAM(NETWORK_DISK_CACHE_REVALIDATION, 1, 10000, 10, EXPONENTIAL, "Total Time spent (ms) during disk cache revalidation")
 HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::Close() on non-main thread (ms)")
 HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE_MAIN_THREAD, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::Close() on the main thread (ms)")
 HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE_INTERNAL, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::CloseInternal() on non-main thread (ms)")