Bug 987829 - HTTP cache v2: make disk cache smart sizing work with the new backend, r=honzab

netwerk/cache/nsCacheService.cpp

@@ -22,6 +22,7 @@
 #include "nsDiskCacheDevice.h"
 #include "nsDiskCacheDeviceSQL.h"
 #include "nsCacheUtils.h"
+#include "../cache2/CacheObserver.h"
 
 #include "nsIObserverService.h"
 #include "nsIPrefService.h"
@@ -3108,6 +3109,10 @@ nsCacheService::SetDiskSmartSize_Locked()
 {
     nsresult rv;
 
+    if (mozilla::net::CacheObserver::UseNewCache()) {
+        return NS_ERROR_NOT_AVAILABLE;
+    }
+
     if (!mObserver->DiskCacheParentDirectory())
         return NS_ERROR_NOT_AVAILABLE;
 

netwerk/cache2/CacheFileIOManager.cpp

@@ -27,6 +27,7 @@
 #include "nsAppDirectoryServiceDefs.h"
 #include "private/pprio.h"
 #include "mozilla/VisualEventTracer.h"
+#include "mozilla/Preferences.h"
 
 // include files for ftruncate (or equivalent)
 #if defined(XP_UNIX)
@@ -43,9 +44,16 @@
 namespace mozilla {
 namespace net {
 
-#define kOpenHandlesLimit      64
-#define kMetadataWriteDelay    5000
-#define kRemoveTrashStartDelay 60000   // in milliseconds
+#define kOpenHandlesLimit        64
+#define kMetadataWriteDelay      5000
+#define kRemoveTrashStartDelay   60000 // in milliseconds
+#define kSmartSizeUpdateInterval 60000 // in milliseconds
+
+#ifdef ANDROID
+const uint32_t kMaxCacheSizeKB = 200*1024; // 200 MB
+#else
+const uint32_t kMaxCacheSizeKB = 350*1024; // 350 MB
+#endif
 
 bool
 CacheFileHandle::DispatchRelease()
@@ -2378,6 +2386,8 @@ CacheFileIOManager::EvictIfOverLimitInternal()
     return NS_OK;
   }
 
+  UpdateSmartCacheSize();
+
   uint32_t cacheUsage;
   rv = CacheIndex::GetCacheSize(&cacheUsage);
   NS_ENSURE_SUCCESS(rv, rv);
@@ -2424,6 +2434,8 @@ CacheFileIOManager::OverLimitEvictionInternal()
     return NS_ERROR_NOT_INITIALIZED;
   }
 
+  UpdateSmartCacheSize();
+
   while (true) {
     uint32_t cacheUsage;
     rv = CacheIndex::GetCacheSize(&cacheUsage);
@@ -3532,6 +3544,116 @@ CacheFileIOManager::SyncRemoveAllCacheFiles()
   }
 }
 
+// Returns default ("smart") size (in KB) of cache, given available disk space
+// (also in KB)
+static uint32_t
+SmartCacheSize(const uint32_t availKB)
+{
+  uint32_t maxSize = kMaxCacheSizeKB;
+
+  if (availKB > 100 * 1024 * 1024) {
+    return maxSize;  // skip computing if we're over 100 GB
+  }
+
+  // Grow/shrink in 10 MB units, deliberately, so that in the common case we
+  // don't shrink cache and evict items every time we startup (it's important
+  // that we don't slow down startup benchmarks).
+  uint32_t sz10MBs = 0;
+  uint32_t avail10MBs = availKB / (1024*10);
+
+  // .5% of space above 25 GB
+  if (avail10MBs > 2500) {
+    sz10MBs += static_cast<uint32_t>((avail10MBs - 2500)*.005);
+    avail10MBs = 2500;
+  }
+  // 1% of space between 7GB -> 25 GB
+  if (avail10MBs > 700) {
+    sz10MBs += static_cast<uint32_t>((avail10MBs - 700)*.01);
+    avail10MBs = 700;
+  }
+  // 5% of space between 500 MB -> 7 GB
+  if (avail10MBs > 50) {
+    sz10MBs += static_cast<uint32_t>((avail10MBs - 50)*.05);
+    avail10MBs = 50;
+  }
+
+#ifdef ANDROID
+  // On Android, smaller/older devices may have very little storage and
+  // device owners may be sensitive to storage footprint: Use a smaller
+  // percentage of available space and a smaller minimum.
+
+  // 20% of space up to 500 MB (10 MB min)
+  sz10MBs += std::max<uint32_t>(1, static_cast<uint32_t>(avail10MBs * .2));
+#else
+  // 40% of space up to 500 MB (50 MB min)
+  sz10MBs += std::max<uint32_t>(5, static_cast<uint32_t>(avail10MBs * .4));
+#endif
+
+  return std::min<uint32_t>(maxSize, sz10MBs * 10 * 1024);
+}
+
+nsresult
+CacheFileIOManager::UpdateSmartCacheSize()
+{
+  MOZ_ASSERT(mIOThread->IsCurrentThread());
+
+  nsresult rv;
+
+  if (!CacheObserver::UseNewCache()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  if (!CacheObserver::SmartCacheSizeEnabled()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  // Wait at least kSmartSizeUpdateInterval before recomputing smart size.
+  static const TimeDuration kUpdateLimit =
+    TimeDuration::FromMilliseconds(kSmartSizeUpdateInterval);
+  if (!mLastSmartSizeTime.IsNull() &&
+      (TimeStamp::NowLoRes() - mLastSmartSizeTime) < kUpdateLimit) {
+    return NS_OK;
+  }
+
+  // Do not compute smart size when cache size is not reliable.
+  bool isUpToDate = false;
+  CacheIndex::IsUpToDate(&isUpToDate);
+  if (!isUpToDate) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  uint32_t cacheUsage;
+  rv = CacheIndex::GetCacheSize(&cacheUsage);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    LOG(("CacheFileIOManager::UpdateSmartCacheSize() - Cannot get cacheUsage! "
+         "[rv=0x%08x]", rv));
+    return rv;
+  }
+
+  int64_t avail;
+  rv = mCacheDirectory->GetDiskSpaceAvailable(&avail);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    // Do not change smart size.
+    LOG(("CacheFileIOManager::UpdateSmartCacheSize() - GetDiskSpaceAvailable() "
+         "failed! [rv=0x%08x]", rv));
+    return rv;
+  }
+
+  mLastSmartSizeTime = TimeStamp::NowLoRes();
+
+  uint32_t smartSize = SmartCacheSize(static_cast<uint32_t>(avail / 1024) +
+                                      cacheUsage);
+
+  if (smartSize == (CacheObserver::DiskCacheCapacity() >> 10)) {
+    // Smart size has not changed.
+    return NS_OK;
+  }
+
+  CacheObserver::SetDiskCacheCapacity(smartSize << 10);
+
+  return NS_OK;
+}
+
 // Memory reporting
 
 namespace { // anon

netwerk/cache2/CacheFileIOManager.h

@@ -359,6 +359,12 @@ private:
   static nsresult CacheIndexStateChanged();
   nsresult CacheIndexStateChangedInternal();
 
+  // Smart size calculation. UpdateSmartCacheSize() must be called on IO thread.
+  // It is called in EvictIfOverLimitInternal() just before we decide whether to
+  // start overlimit eviction or not and also in OverLimitEvictionInternal()
+  // before we start an eviction loop.
+  nsresult UpdateSmartCacheSize();
+
   // Memory reporting (private part)
   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 
@@ -380,6 +386,7 @@ private:
   nsCOMPtr<nsIDirectoryEnumerator>     mTrashDirEnumerator;
   nsTArray<nsCString>                  mFailedTrashDirs;
   nsRefPtr<CacheFileContextEvictor>    mContextEvictor;
+  TimeStamp                            mLastSmartSizeTime;
 };
 
 } // net

netwerk/cache2/CacheObserver.cpp

@@ -53,6 +53,9 @@ int32_t CacheObserver::sAutoMemoryCacheCapacity = -1;
 static uint32_t const kDefaultDiskCacheCapacity = 250 * 1024; // 250 MB
 uint32_t CacheObserver::sDiskCacheCapacity = kDefaultDiskCacheCapacity;
 
+static bool const kDefaultSmartCacheSizeEnabled = false;
+bool CacheObserver::sSmartCacheSizeEnabled = kDefaultSmartCacheSizeEnabled;
+
 static uint32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
 uint32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
 
@@ -133,6 +136,8 @@ CacheObserver::AttachToPreferences()
 
   mozilla::Preferences::AddUintVarCache(
     &sDiskCacheCapacity, "browser.cache.disk.capacity", kDefaultDiskCacheCapacity);
+  mozilla::Preferences::AddBoolVarCache(
+    &sSmartCacheSizeEnabled, "browser.cache.disk.smart_size.enabled", kDefaultSmartCacheSizeEnabled);
   mozilla::Preferences::AddIntVarCache(
     &sMemoryCacheCapacity, "browser.cache.memory.capacity", kDefaultMemoryCacheCapacity);
 
@@ -273,6 +278,32 @@ bool const CacheObserver::UseNewCache()
   return true;
 }
 
+// static
+void
+CacheObserver::SetDiskCacheCapacity(uint32_t aCapacity)
+{
+  sDiskCacheCapacity = aCapacity >> 10;
+
+  if (!sSelf) {
+    return;
+  }
+
+  if (NS_IsMainThread()) {
+    sSelf->StoreDiskCacheCapacity();
+  } else {
+    nsCOMPtr<nsIRunnable> event =
+      NS_NewRunnableMethod(sSelf, &CacheObserver::StoreDiskCacheCapacity);
+    NS_DispatchToMainThread(event);
+  }
+}
+
+void
+CacheObserver::StoreDiskCacheCapacity()
+{
+  mozilla::Preferences::SetInt("browser.cache.disk.capacity",
+                               sDiskCacheCapacity);
+}
+
 // static
 void CacheObserver::ParentDirOverride(nsIFile** aDir)
 {

netwerk/cache2/CacheObserver.h

@@ -17,7 +17,7 @@ namespace net {
 class CacheObserver : public nsIObserver
                     , public nsSupportsWeakReference
 {
-  NS_DECL_ISUPPORTS
+  NS_DECL_THREADSAFE_ISUPPORTS
   NS_DECL_NSIOBSERVER
 
   virtual ~CacheObserver() {}
@@ -37,6 +37,9 @@ class CacheObserver : public nsIObserver
   static uint32_t const MemoryCacheCapacity(); // result in bytes.
   static uint32_t const DiskCacheCapacity() // result in bytes.
     { return sDiskCacheCapacity << 10; }
+  static void SetDiskCacheCapacity(uint32_t); // parameter in bytes.
+  static bool const SmartCacheSizeEnabled()
+    { return sSmartCacheSizeEnabled; }
   static uint32_t const MaxMemoryEntrySize() // result in bytes.
     { return sMaxMemoryEntrySize << 10; }
   static uint32_t const MaxDiskEntrySize() // result in bytes.
@@ -56,6 +59,7 @@ class CacheObserver : public nsIObserver
 private:
   static CacheObserver* sSelf;
 
+  void StoreDiskCacheCapacity();
   void AttachToPreferences();
   void SchduleAutoDelete();
 
@@ -66,6 +70,7 @@ private:
   static int32_t sMemoryCacheCapacity;
   static int32_t sAutoMemoryCacheCapacity;
   static uint32_t sDiskCacheCapacity;
+  static bool sSmartCacheSizeEnabled;
   static uint32_t sMaxMemoryEntrySize;
   static uint32_t sMaxDiskEntrySize;
   static uint32_t sCompressionLevel;