Bug 913822 - Shutdown I/O time limit for HTTP cache. r=michal

modules/libpref/init/all.js

@@ -89,6 +89,11 @@ pref("browser.cache.disk.preload_chunk_count", 4); // 1 MB of read ahead
 // The half life used to re-compute cache entries frecency in hours.
 pref("browser.cache.frecency_half_life_hours", 6);
 
+// Number of seconds the cache spends writting pending data and closing files
+// after the shutdown has been signalled.  Past that time data are never written
+// and files are left open given up to the OS to do the cleanup.
+pref("browser.cache.max_shutdown_io_lag", 2);
+
 pref("browser.cache.offline.enable",           true);
 // enable offline apps by default, disable prompt
 pref("offline-apps.allow_by_default",          true);

netwerk/cache2/CacheFileIOManager.cpp

@@ -536,6 +536,7 @@ public:
   ShutdownEvent(mozilla::Mutex *aLock, mozilla::CondVar *aCondVar)
     : mLock(aLock)
     , mCondVar(aCondVar)
+    , mPrepare(true)
   {
     MOZ_COUNT_CTOR(ShutdownEvent);
   }
@@ -549,6 +550,21 @@ protected:
 public:
   NS_IMETHOD Run()
   {
+    if (mPrepare) {
+      MOZ_ASSERT(CacheFileIOManager::gInstance->mIOThread->IsCurrentThread());
+
+      mPrepare = false;
+
+      // This event is first posted to the XPCOM level (executed ASAP) of the IO thread
+      // and sets the timestamp of the shutdown start.  This will cause some operations
+      // to be bypassed when due (actually leak most of the open files).
+      CacheFileIOManager::gInstance->mShutdownDemandedTime = TimeStamp::NowLoRes();
+
+      // Redispatch to the right level to proceed with shutdown.
+      CacheFileIOManager::gInstance->mIOThread->Dispatch(this, CacheIOThread::CLOSE);
+      return NS_OK;
+    }
+
     MutexAutoLock lock(*mLock);
 
     CacheFileIOManager::gInstance->ShutdownInternal();
@@ -560,6 +576,7 @@ public:
 protected:
   mozilla::Mutex   *mLock;
   mozilla::CondVar *mCondVar;
+  bool              mPrepare;
 };
 
 class OpenFileEvent : public nsRunnable {
@@ -697,7 +714,13 @@ public:
     nsresult rv;
 
     if (mHandle->IsClosed()) {
-      rv = NS_ERROR_NOT_INITIALIZED;
+      // We usually get here only after the internal shutdown
+      // (i.e. mShuttingDown == true).  Pretend write has succeeded
+      // to avoid any past-shutdown file dooming.
+      rv = (CacheFileIOManager::gInstance->IsPastShutdownIOLag() ||
+            CacheFileIOManager::gInstance->mShuttingDown)
+        ? NS_OK
+        : NS_ERROR_NOT_INITIALIZED;
     } else {
       rv = CacheFileIOManager::gInstance->WriteInternal(
           mHandle, mOffset, mBuf, mCount, mValidate, mTruncate);
@@ -1148,7 +1171,9 @@ CacheFileIOManager::Shutdown()
     MutexAutoLock autoLock(lock);
     RefPtr<ShutdownEvent> ev = new ShutdownEvent(&lock, &condVar);
     DebugOnly<nsresult> rv;
-    rv = gInstance->mIOThread->Dispatch(ev, CacheIOThread::CLOSE);
+    nsCOMPtr<nsIEventTarget> ioTarget = gInstance->mIOThread->Target();
+    MOZ_ASSERT(ioTarget);
+    rv = ioTarget->Dispatch(ev, nsIEventTarget::DISPATCH_NORMAL);
     MOZ_ASSERT(NS_SUCCEEDED(rv));
     condVar.Wait();
   }
@@ -1240,6 +1265,26 @@ CacheFileIOManager::ShutdownInternal()
   return NS_OK;
 }
 
+bool
+CacheFileIOManager::IsPastShutdownIOLag()
+{
+#ifdef DEBUG
+  return false;
+#endif
+
+  if (mShutdownDemandedTime.IsNull()) {
+    return false;
+  }
+
+  TimeDuration const& preferredIOLag = CacheObserver::MaxShutdownIOLag();
+  if (preferredIOLag < TimeDuration(0)) {
+    return false;
+  }
+
+  TimeDuration currentIOLag = TimeStamp::NowLoRes() - mShutdownDemandedTime;
+  return currentIOLag > preferredIOLag;
+}
+
 // static
 nsresult
 CacheFileIOManager::OnProfile()
@@ -1925,6 +1970,13 @@ CacheFileIOManager::WriteInternal(CacheFileHandle *aHandle, int64_t aOffset,
 
   nsresult rv;
 
+  if (IsPastShutdownIOLag()) {
+    LOG(("  past the shutdown I/O lag, nothing written"));
+    // Pretend the write has succeeded, otherwise upper layers will doom
+    // the file and we end up with I/O anyway.
+    return NS_OK;
+  }
+
   if (!aHandle->mFileExists) {
     rv = CreateFile(aHandle);
     NS_ENSURE_SUCCESS(rv, rv);
@@ -2082,7 +2134,7 @@ CacheFileIOManager::DoomFileInternal(CacheFileHandle *aHandle,
   if (aHandle->mFileExists) {
     // we need to move the current file to the doomed directory
     if (aHandle->mFD) {
-      ReleaseNSPRHandleInternal(aHandle);
+      ReleaseNSPRHandleInternal(aHandle, true);
     }
 
     // find unused filename
@@ -2229,7 +2281,8 @@ CacheFileIOManager::ReleaseNSPRHandle(CacheFileHandle *aHandle)
 }
 
 nsresult
-CacheFileIOManager::ReleaseNSPRHandleInternal(CacheFileHandle *aHandle)
+CacheFileIOManager::ReleaseNSPRHandleInternal(CacheFileHandle *aHandle,
+                                              bool aIgnoreShutdownLag)
 {
   LOG(("CacheFileIOManager::ReleaseNSPRHandleInternal() [handle=%p]", aHandle));
 
@@ -2240,7 +2293,17 @@ CacheFileIOManager::ReleaseNSPRHandleInternal(CacheFileHandle *aHandle)
   found = mHandlesByLastUsed.RemoveElement(aHandle);
   MOZ_ASSERT(found);
 
-  PR_Close(aHandle->mFD);
+  if (aIgnoreShutdownLag || !IsPastShutdownIOLag()) {
+    PR_Close(aHandle->mFD);
+  } else {
+    // Pretend this file has been validated (the metadata has been written)
+    // to prevent removal I/O on this apparently used file.  The entry will
+    // never be used, since it doesn't have correct metadata, thus we don't
+    // need to worry about removing it.
+    aHandle->mInvalid = false;
+    LOG(("  past the shutdown I/O lag, leaking file handle"));
+  }
+
   aHandle->mFD = nullptr;
 
   return NS_OK;
@@ -2538,7 +2601,7 @@ CacheFileIOManager::RenameFileInternal(CacheFileHandle *aHandle,
   }
 
   if (aHandle->mFD) {
-    ReleaseNSPRHandleInternal(aHandle);
+    ReleaseNSPRHandleInternal(aHandle, true);
   }
 
   rv = aHandle->mFile->MoveToNative(nullptr, aNewName);
@@ -3731,7 +3794,7 @@ CacheFileIOManager::OpenNSPRHandle(CacheFileHandle *aHandle, bool aCreate)
 
   if (mHandlesByLastUsed.Length() == kOpenHandlesLimit) {
     // close handle that hasn't been used for the longest time
-    rv = ReleaseNSPRHandleInternal(mHandlesByLastUsed[0]);
+    rv = ReleaseNSPRHandleInternal(mHandlesByLastUsed[0], true);
     NS_ENSURE_SUCCESS(rv, rv);
   }
 

netwerk/cache2/CacheFileIOManager.h

@@ -382,7 +382,8 @@ private:
   nsresult DoomFileInternal(CacheFileHandle *aHandle,
                             PinningDoomRestriction aPinningStatusRestriction = NO_RESTRICTION);
   nsresult DoomFileByKeyInternal(const SHA1Sum::Hash *aHash);
-  nsresult ReleaseNSPRHandleInternal(CacheFileHandle *aHandle);
+  nsresult ReleaseNSPRHandleInternal(CacheFileHandle *aHandle,
+                                     bool aIgnoreShutdownLag = false);
   nsresult TruncateSeekSetEOFInternal(CacheFileHandle *aHandle,
                                       int64_t aTruncatePos, int64_t aEOFPos);
   nsresult RenameFileInternal(CacheFileHandle *aHandle,
@@ -429,11 +430,18 @@ private:
   // before we start an eviction loop.
   nsresult UpdateSmartCacheSize(int64_t aFreeSpace);
 
+  // May return true after shutdown only when time for flushing all data
+  // has already passed.
+  bool IsPastShutdownIOLag();
+
   // Memory reporting (private part)
   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 
   static CacheFileIOManager           *gInstance;
   TimeStamp                            mStartTime;
+  // Shutdown time stamp, accessed only on the I/O thread.  Used to bypass
+  // I/O after a certain time pass the shutdown has been demanded.
+  TimeStamp                            mShutdownDemandedTime;
   bool                                 mShuttingDown;
   RefPtr<CacheIOThread>                mIOThread;
   nsCOMPtr<nsIFile>                    mCacheDirectory;

netwerk/cache2/CacheObserver.cpp

@@ -11,6 +11,7 @@
 #include "nsIObserverService.h"
 #include "mozilla/Services.h"
 #include "mozilla/Preferences.h"
+#include "mozilla/TimeStamp.h"
 #include "nsServiceManagerUtils.h"
 #include "prsystem.h"
 #include <time.h>
@@ -91,6 +92,9 @@ bool CacheObserver::sCacheFSReported = kDefaultCacheFSReported;
 static bool kDefaultHashStatsReported = false;
 bool CacheObserver::sHashStatsReported = kDefaultHashStatsReported;
 
+static int32_t const kDefaultMaxShutdownIOLag = 2; // seconds
+int32_t CacheObserver::sMaxShutdownIOLag = kDefaultMaxShutdownIOLag;
+
 NS_IMPL_ISUPPORTS(CacheObserver,
                   nsIObserver,
                   nsISupportsWeakReference)
@@ -237,6 +241,9 @@ CacheObserver::AttachToPreferences()
     &sSanitizeOnShutdown, "privacy.sanitize.sanitizeOnShutdown", kDefaultSanitizeOnShutdown);
   mozilla::Preferences::AddBoolVarCache(
     &sClearCacheOnShutdown, "privacy.clearOnShutdown.cache", kDefaultClearCacheOnShutdown);
+
+  mozilla::Preferences::AddIntVarCache(
+    &sMaxShutdownIOLag, "browser.cache.max_shutdown_io_lag", kDefaultMaxShutdownIOLag);
 }
 
 // static
@@ -466,6 +473,13 @@ bool CacheObserver::EntryIsTooBig(int64_t aSize, bool aUsingDisk)
   return false;
 }
 
+// static
+TimeDuration const& CacheObserver::MaxShutdownIOLag()
+{
+  static TimeDuration period = TimeDuration::FromSeconds(sMaxShutdownIOLag);
+  return period;
+}
+
 NS_IMETHODIMP
 CacheObserver::Observe(nsISupports* aSubject,
                        const char* aTopic,

netwerk/cache2/CacheObserver.h

@@ -71,6 +71,8 @@ class CacheObserver : public nsIObserver
 
   static bool EntryIsTooBig(int64_t aSize, bool aUsingDisk);
 
+  static TimeDuration const& MaxShutdownIOLag();
+
 private:
   static CacheObserver* sSelf;
 
@@ -101,6 +103,7 @@ private:
   static bool sClearCacheOnShutdown;
   static bool sCacheFSReported;
   static bool sHashStatsReported;
+  static int32_t sMaxShutdownIOLag;
 
   // Non static properties, accessible via sSelf
   nsCOMPtr<nsIFile> mCacheParentDirectoryOverride;