Bug 924116 - HTTP cache v2: persists frecency and experiment with half-life, r=michal

netwerk/cache2/CacheEntry.cpp

@@ -371,9 +371,16 @@ NS_IMETHODIMP CacheEntry::OnFileReady(nsresult aResult, bool aIsNew)
 
   mFileStatus = aResult;
 
-  if (mState == READY)
+  if (mState == READY) {
     mHasData = true;
 
+    uint32_t frecency;
+    mFile->GetFrecency(&frecency);
+    // mFrecency is held in a double to increase computance precision.
+    // It is ok to persist frecency only as a uint32 with some math involved.
+    mFrecency = INT2FRECENCY(frecency);
+  }
+
   InvokeCallbacks();
   return NS_OK;
 }
@@ -1459,8 +1466,8 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync)
     #define M_LN2 0.69314718055994530942
     #endif
 
-    // Half-life is 90 days.
-    static double const half_life = 90.0 * (24 * 60 * 60);
+    // Half-life is dynamic, in seconds.
+     static double half_life = CacheObserver::HalfLifeSeconds();
     // Must convert from seconds to milliseconds since PR_Now() gives usecs.
     static double const decay = (M_LN2 / half_life) / static_cast<double>(PR_USEC_PER_SEC);
 
@@ -1475,6 +1482,12 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync)
       mFrecency = log(exp(mFrecency - now_decay) + 1) + now_decay;
     }
     LOG(("CacheEntry FRECENCYUPDATE [this=%p, frecency=%1.10f]", this, mFrecency));
+
+    // Because CacheFile::Set*() are not thread-safe to use (uses WeakReference that
+    // is not thread-safe) we must post to the main thread...
+    nsRefPtr<nsRunnableMethod<CacheEntry> > event =
+      NS_NewRunnableMethod(this, &CacheEntry::StoreFrecency);
+    NS_DispatchToMainThread(event);
   }
 
   if (aOperations & Ops::REGISTER) {
@@ -1497,6 +1510,14 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync)
   }
 }
 
+void CacheEntry::StoreFrecency()
+{
+  // No need for thread safety over mFrecency, it will be rewriten
+  // correctly on following invocation if broken by concurrency.
+  MOZ_ASSERT(NS_IsMainThread());
+  mFile->SetFrecency(FRECENCY2INT(mFrecency));
+}
+
 // CacheOutputCloseListener
 
 CacheOutputCloseListener::CacheOutputCloseListener(CacheEntry* aEntry)

netwerk/cache2/CacheEntry.h

@@ -219,6 +219,7 @@ private:
   // When executed on the management thread directly, the operation(s)
   // is (are) executed immediately.
   void BackgroundOp(uint32_t aOperation, bool aForceAsync = false);
+  void StoreFrecency();
 
   already_AddRefed<CacheEntryHandle> ReopenTruncated(nsICacheEntryOpenCallback* aCallback);
   void TransferCallbacks(CacheEntry & aFromEntry);

netwerk/cache2/CacheFile.cpp

@@ -1034,6 +1034,27 @@ CacheFile::GetLastModified(uint32_t *_retval)
   return mMetadata->GetLastModified(_retval);
 }
 
+nsresult
+CacheFile::SetFrecency(uint32_t aFrecency)
+{
+  CacheFileAutoLock lock(this);
+  MOZ_ASSERT(mMetadata);
+  NS_ENSURE_TRUE(mMetadata, NS_ERROR_UNEXPECTED);
+
+  PostWriteTimer();
+  return mMetadata->SetFrecency(aFrecency);
+}
+
+nsresult
+CacheFile::GetFrecency(uint32_t *_retval)
+{
+  CacheFileAutoLock lock(this);
+  MOZ_ASSERT(mMetadata);
+  NS_ENSURE_TRUE(mMetadata, NS_ERROR_UNEXPECTED);
+
+  return mMetadata->GetFrecency(_retval);
+}
+
 nsresult
 CacheFile::GetLastFetched(uint32_t *_retval)
 {

netwerk/cache2/CacheFile.h

@@ -92,6 +92,8 @@ public:
   nsresult GetExpirationTime(uint32_t *_retval);
   nsresult SetLastModified(uint32_t aLastModified);
   nsresult GetLastModified(uint32_t *_retval);
+  nsresult SetFrecency(uint32_t aFrecency);
+  nsresult GetFrecency(uint32_t *_retval);
   nsresult GetLastFetched(uint32_t *_retval);
   nsresult GetFetchCount(uint32_t *_retval);
 

netwerk/cache2/CacheFileMetadata.cpp

@@ -420,6 +420,24 @@ CacheFileMetadata::GetLastModified(uint32_t *_retval)
   return NS_OK;
 }
 
+nsresult
+CacheFileMetadata::SetFrecency(uint32_t aFrecency)
+{
+  LOG(("CacheFileMetadata::SetFrecency() [this=%p, frecency=%f]",
+       this, (double)aFrecency));
+
+  MarkDirty();
+  mMetaHdr.mFrecency = aFrecency;
+  return NS_OK;
+}
+
+nsresult
+CacheFileMetadata::GetFrecency(uint32_t *_retval)
+{
+  *_retval = mMetaHdr.mFrecency;
+  return NS_OK;
+}
+
 nsresult
 CacheFileMetadata::GetLastFetched(uint32_t *_retval)
 {

netwerk/cache2/CacheFileMetadata.h

@@ -8,16 +8,28 @@
 #include "CacheFileIOManager.h"
 #include "CacheStorageService.h"
 #include "CacheHashUtils.h"
+#include "CacheObserver.h"
 #include "nsAutoPtr.h"
 #include "nsString.h"
 
 namespace mozilla {
 namespace net {
 
+// By multiplying with the current half-life we convert the frecency
+// to time independent of half-life value.  The range fits 32bits.
+// When decay time changes on next run of the browser, we convert
+// the frecency value to a correct internal representation again.
+// It might not be 100% accurate, but for the purpose it suffice.
+#define FRECENCY2INT(aFrecency) \
+  ((uint32_t)(aFrecency * CacheObserver::HalfLifeSeconds()))
+#define INT2FRECENCY(aInt) \
+  ((double)(aInt) / (double)CacheObserver::HalfLifeSeconds())
+
 typedef struct {
   uint32_t        mFetchCount;
   uint32_t        mLastFetched;
   uint32_t        mLastModified;
+  uint32_t        mFrecency;
   uint32_t        mExpirationTime;
   uint32_t        mKeySize;
 } CacheFileMetadataHeader;
@@ -74,6 +86,8 @@ public:
   nsresult GetExpirationTime(uint32_t *_retval);
   nsresult SetLastModified(uint32_t aLastModified);
   nsresult GetLastModified(uint32_t *_retval);
+  nsresult SetFrecency(uint32_t aFrecency);
+  nsresult GetFrecency(uint32_t *_retval);
   nsresult GetLastFetched(uint32_t *_retval);
   nsresult GetFetchCount(uint32_t *_retval);
 

netwerk/cache2/CacheObserver.cpp

@@ -13,6 +13,7 @@
 #include "mozilla/Services.h"
 #include "mozilla/Preferences.h"
 #include "nsServiceManagerUtils.h"
+#include <time.h>
 
 namespace mozilla {
 namespace net {
@@ -25,6 +26,12 @@ uint32_t CacheObserver::sMemoryLimit = kDefaultMemoryLimit;
 static uint32_t const kDefaultUseNewCache = 0; // Don't use the new cache by default
 uint32_t CacheObserver::sUseNewCache = kDefaultUseNewCache;
 
+static int32_t const kDefaultHalfLifeExperiment = -1; // Disabled
+int32_t CacheObserver::sHalfLifeExperiment = kDefaultHalfLifeExperiment;
+
+static uint32_t const kDefaultHalfLifeHours = 6; // 6 hours
+uint32_t CacheObserver::sHalfLifeHours = kDefaultHalfLifeHours;
+
 NS_IMPL_ISUPPORTS2(CacheObserver,
                    nsIObserver,
                    nsISupportsWeakReference)
@@ -73,6 +80,41 @@ CacheObserver::AttachToPreferences()
     &sMemoryLimit, "browser.cache.memory_limit", kDefaultMemoryLimit);
   mozilla::Preferences::AddUintVarCache(
     &sUseNewCache, "browser.cache.use_new_backend", kDefaultUseNewCache);
+
+  sHalfLifeExperiment = mozilla::Preferences::GetInt(
+    "browser.cache.frecency_experiment", kDefaultHalfLifeExperiment);
+
+  if (sHalfLifeExperiment == 0) {
+    // The experiment has not yet been initialized, do it now
+    // Store the experiemnt value, since we need it not to change between
+    // browser sessions.
+    srand(time(NULL));
+    sHalfLifeExperiment = (rand() % 4) + 1;
+    mozilla::Preferences::SetInt(
+      "browser.cache.frecency_experiment", sHalfLifeExperiment);
+  }
+
+  switch (sHalfLifeExperiment) {
+  case 1: // The experiment is engaged
+    sHalfLifeHours = 6;
+    break;
+  case 2:
+    sHalfLifeHours = 24;
+    break;
+  case 3:
+    sHalfLifeHours = 7 * 24;
+    break;
+  case 4:
+    sHalfLifeHours = 50 * 24;
+    break;
+
+  case -1:
+  default: // The experiment is off or broken
+    sHalfLifeExperiment = -1;
+    sHalfLifeHours = std::max(1U, std::min(1440U, mozilla::Preferences::GetUint(
+      "browser.cache.frecency_half_life_hours", kDefaultHalfLifeHours)));
+    break;
+  }
 }
 
 // static

netwerk/cache2/CacheObserver.h

@@ -28,6 +28,10 @@ class CacheObserver : public nsIObserver
   static uint32_t const MemoryLimit() // <0.5MB,1024MB>, result in bytes.
     { return std::max(512U, std::min(1048576U, sMemoryLimit)) << 10; }
   static bool const UseNewCache();
+  static uint32_t const HalfLifeSeconds()
+    { return sHalfLifeHours * 60 * 60; }
+  static int32_t const HalfLifeExperiment()
+    { return sHalfLifeExperiment; }
 
 private:
   static CacheObserver* sSelf;
@@ -36,6 +40,8 @@ private:
 
   static uint32_t sMemoryLimit;
   static uint32_t sUseNewCache;
+  static uint32_t sHalfLifeHours;
+  static int32_t sHalfLifeExperiment;
 };
 
 } // net

netwerk/protocol/http/nsHttpChannel.cpp

@@ -92,6 +92,12 @@ AccumulateCacheHitTelemetry(CacheDisposition hitOrMiss)
     }
     else {
         Telemetry::Accumulate(Telemetry::HTTP_CACHE_DISPOSITION_2_V2, hitOrMiss);
+
+        int32_t experiment = CacheObserver::HalfLifeExperiment();
+        if (experiment > 0 && hitOrMiss == kCacheMissed) {
+            Telemetry::Accumulate(Telemetry::HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT,
+                                  experiment - 1);
+        }
     }
 }
 

toolkit/components/telemetry/Histograms.json

@@ -1448,6 +1448,12 @@
     "n_values": 5,
     "description": "HTTP Disk Cache Hit, Reval, Failed-Reval, Miss"
   },
+  "HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": 4,
+    "description": "HTTP Cache v2 Miss by half-life value (6h, 1d, 7d, 50d)"
+  },
   "HTTP_MEMORY_CACHE_DISPOSITION_2": {
     "expires_in_version": "never",
     "kind": "enumerated",