From dc3cc5f86265e90289f9f7b19693e573551ea195 Mon Sep 17 00:00:00 2001 From: Michal Novotny Date: Tue, 18 Feb 2014 18:26:48 +0100 Subject: [PATCH] Bug 913808 - Evict entries from the disk cache when cache limit is reached, r=honzab --- netwerk/cache2/CacheEntriesEnumerator.cpp | 90 --------- netwerk/cache2/CacheEntriesEnumerator.h | 2 - netwerk/cache2/CacheEntry.cpp | 12 +- netwerk/cache2/CacheEntry.h | 1 + netwerk/cache2/CacheFile.cpp | 74 ++++---- netwerk/cache2/CacheFile.h | 3 +- netwerk/cache2/CacheFileIOManager.cpp | 221 +++++++++++++++++++--- netwerk/cache2/CacheFileIOManager.h | 8 +- netwerk/cache2/CacheFileMetadata.cpp | 199 ++++--------------- netwerk/cache2/CacheFileMetadata.h | 7 +- netwerk/cache2/CacheFileUtils.cpp | 115 +++++++++++ netwerk/cache2/CacheFileUtils.h | 27 +++ netwerk/cache2/CacheIOThread.h | 2 +- netwerk/cache2/CacheIndex.cpp | 102 +++++++++- netwerk/cache2/CacheIndex.h | 15 +- netwerk/cache2/CacheStorageService.cpp | 99 ++++------ netwerk/cache2/CacheStorageService.h | 9 + netwerk/cache2/moz.build | 1 + 18 files changed, 592 insertions(+), 395 deletions(-) create mode 100644 netwerk/cache2/CacheFileUtils.cpp create mode 100644 netwerk/cache2/CacheFileUtils.h diff --git a/netwerk/cache2/CacheEntriesEnumerator.cpp b/netwerk/cache2/CacheEntriesEnumerator.cpp index 4360535914b..f8a457a040e 100644 --- a/netwerk/cache2/CacheEntriesEnumerator.cpp +++ b/netwerk/cache2/CacheEntriesEnumerator.cpp @@ -80,80 +80,6 @@ bool CacheEntriesEnumerator::HasMore() return !!mCurrentFile; } -namespace { // anon - -class FileConsumer : public CacheFileListener -{ -public: - NS_DECL_THREADSAFE_ISUPPORTS - - nsresult Init(nsCSubstring const & aKey, - CacheEntriesEnumeratorCallback* aCallback); - - virtual ~FileConsumer() {} - -private: - NS_IMETHOD OnFileReady(nsresult aResult, bool aIsNew); - NS_IMETHOD OnFileDoomed(nsresult aResult) { return NS_OK; } - - nsRefPtr mFile; - nsRefPtr mCallback; -}; - -nsresult FileConsumer::Init(const nsCSubstring &aKey, - CacheEntriesEnumeratorCallback *aCallback) -{ - mCallback = aCallback; - - mFile = new CacheFile(); - nsresult rv = mFile->Init(aKey, false, false, false, true, this); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; -} - -NS_IMETHODIMP FileConsumer::OnFileReady(nsresult aResult, bool aIsNew) -{ - //MOZ_ASSERT(!aIsNew); - - if (NS_FAILED(aResult)) { - mCallback->OnFile(nullptr); - } - else { - mCallback->OnFile(mFile); - } - - return NS_OK; -} - -NS_IMPL_ISUPPORTS1(FileConsumer, CacheFileListener); - -} // anon - -nsresult CacheEntriesEnumerator::GetNextCacheFile(CacheEntriesEnumeratorCallback* aCallback) -{ -#ifdef DEBUG - MOZ_ASSERT(mThreadCheck == NS_GetCurrentThread()); -#endif - - nsresult rv; - - NS_ENSURE_TRUE(mCurrentFile, NS_ERROR_UNEXPECTED); - - nsAutoCString key; - rv = mCurrentFile->GetNativeLeafName(key); - - mCurrentFile = nullptr; - - NS_ENSURE_SUCCESS(rv, rv); - - nsRefPtr consumer = new FileConsumer(); - rv = consumer->Init(key, aCallback); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; -} - nsresult CacheEntriesEnumerator::GetNextFile(nsIFile** aFile) { #ifdef DEBUG @@ -166,21 +92,5 @@ nsresult CacheEntriesEnumerator::GetNextFile(nsIFile** aFile) return NS_OK; } -nsresult CacheEntriesEnumerator::GetCacheFileFromFile(nsIFile* aFile, - CacheEntriesEnumeratorCallback* aCallback) -{ - nsresult rv; - - nsAutoCString key; - rv = aFile->GetNativeLeafName(key); - NS_ENSURE_SUCCESS(rv, rv); - - nsRefPtr consumer = new FileConsumer(); - rv = consumer->Init(key, aCallback); - NS_ENSURE_SUCCESS(rv, rv); - - return NS_OK; -} - } // net } // mozilla diff --git a/netwerk/cache2/CacheEntriesEnumerator.h b/netwerk/cache2/CacheEntriesEnumerator.h index d9371e10836..544c66d88d8 100644 --- a/netwerk/cache2/CacheEntriesEnumerator.h +++ b/netwerk/cache2/CacheEntriesEnumerator.h @@ -30,9 +30,7 @@ public: ~CacheEntriesEnumerator(); bool HasMore(); - nsresult GetNextCacheFile(CacheEntriesEnumeratorCallback* aCallback); nsresult GetNextFile(nsIFile** aFile); - nsresult GetCacheFileFromFile(nsIFile* aFile, CacheEntriesEnumeratorCallback* aCallback); protected: friend class CacheFileIOManager; diff --git a/netwerk/cache2/CacheEntry.cpp b/netwerk/cache2/CacheEntry.cpp index 1aa18e90fb4..ef58679687a 100644 --- a/netwerk/cache2/CacheEntry.cpp +++ b/netwerk/cache2/CacheEntry.cpp @@ -319,7 +319,6 @@ bool CacheEntry::Load(bool aTruncate, bool aPriority) aTruncate, !mUseDisk, aPriority, - false /* key is not a hash */, directLoad ? nullptr : this); } @@ -837,6 +836,17 @@ bool CacheEntry::IsReferenced() const return mHandlersCount > 0; } +bool CacheEntry::IsFileDoomed() +{ + mozilla::MutexAutoLock lock(mLock); + + if (NS_SUCCEEDED(mFileStatus)) { + return mFile->IsDoomed(); + } + + return false; +} + uint32_t CacheEntry::GetMetadataMemoryConsumption() { NS_ENSURE_SUCCESS(mFileStatus, 0); diff --git a/netwerk/cache2/CacheEntry.h b/netwerk/cache2/CacheEntry.h index 9f9a71b6c8a..bd5badc6284 100644 --- a/netwerk/cache2/CacheEntry.h +++ b/netwerk/cache2/CacheEntry.h @@ -70,6 +70,7 @@ public: bool UsingDisk() const; bool SetUsingDisk(bool aUsingDisk); bool IsReferenced() const; + bool IsFileDoomed(); // Methods for entry management (eviction from memory), // called only on the management thread. diff --git a/netwerk/cache2/CacheFile.cpp b/netwerk/cache2/CacheFile.cpp index fd3891d92d3..c381b1f2d6a 100644 --- a/netwerk/cache2/CacheFile.cpp +++ b/netwerk/cache2/CacheFile.cpp @@ -195,19 +195,15 @@ CacheFile::Init(const nsACString &aKey, bool aCreateNew, bool aMemoryOnly, bool aPriority, - bool aKeyIsHash, CacheFileListener *aCallback) { MOZ_ASSERT(!mListener); MOZ_ASSERT(!mHandle); - MOZ_ASSERT(!(aCreateNew && aKeyIsHash)); - MOZ_ASSERT(!(aMemoryOnly && aKeyIsHash)); nsresult rv; mKey = aKey; mMemoryOnly = aMemoryOnly; - mKeyIsHash = aKeyIsHash; LOG(("CacheFile::Init() [this=%p, key=%s, createNew=%d, memoryOnly=%d, " "listener=%p]", this, mKey.get(), aCreateNew, aMemoryOnly, aCallback)); @@ -215,7 +211,6 @@ CacheFile::Init(const nsACString &aKey, if (mMemoryOnly) { MOZ_ASSERT(!aCallback); - MOZ_ASSERT(!mKeyIsHash); mMetadata = new CacheFileMetadata(mKey); mReady = true; mDataSize = mMetadata->Offset(); @@ -228,7 +223,6 @@ CacheFile::Init(const nsACString &aKey, flags = CacheFileIOManager::CREATE_NEW; // make sure we can use this entry immediately - MOZ_ASSERT(!mKeyIsHash); mMetadata = new CacheFileMetadata(mKey); mReady = true; mDataSize = mMetadata->Offset(); @@ -236,37 +230,33 @@ CacheFile::Init(const nsACString &aKey, else { flags = CacheFileIOManager::CREATE; - if (!mKeyIsHash) { - // Have a look into index and change to CREATE_NEW when we are sure - // that the entry does not exist. - CacheIndex::EntryStatus status; - rv = CacheIndex::HasEntry(mKey, &status); - if (status == CacheIndex::DOES_NOT_EXIST) { - LOG(("CacheFile::Init() - Forcing CREATE_NEW flag since we don't have" - " this entry according to index")); - flags = CacheFileIOManager::CREATE_NEW; + // Have a look into index and change to CREATE_NEW when we are sure + // that the entry does not exist. + CacheIndex::EntryStatus status; + rv = CacheIndex::HasEntry(mKey, &status); + if (status == CacheIndex::DOES_NOT_EXIST) { + LOG(("CacheFile::Init() - Forcing CREATE_NEW flag since we don't have" + " this entry according to index")); + flags = CacheFileIOManager::CREATE_NEW; - // make sure we can use this entry immediately - mMetadata = new CacheFileMetadata(mKey); - mReady = true; - mDataSize = mMetadata->Offset(); + // make sure we can use this entry immediately + mMetadata = new CacheFileMetadata(mKey); + mReady = true; + mDataSize = mMetadata->Offset(); - // Notify callback now and don't store it in mListener, no further - // operation can change the result. - nsRefPtr ev; - ev = new NotifyCacheFileListenerEvent(aCallback, NS_OK, true); - rv = NS_DispatchToCurrentThread(ev); - NS_ENSURE_SUCCESS(rv, rv); + // Notify callback now and don't store it in mListener, no further + // operation can change the result. + nsRefPtr ev; + ev = new NotifyCacheFileListenerEvent(aCallback, NS_OK, true); + rv = NS_DispatchToCurrentThread(ev); + NS_ENSURE_SUCCESS(rv, rv); - aCallback = nullptr; - } + aCallback = nullptr; } } if (aPriority) flags |= CacheFileIOManager::PRIORITY; - if (aKeyIsHash) - flags |= CacheFileIOManager::NOHASH; mOpeningFile = true; mListener = aCallback; @@ -290,7 +280,6 @@ CacheFile::Init(const nsACString &aKey, "initializing entry as memory-only. [this=%p]", this)); mMemoryOnly = true; - MOZ_ASSERT(!mKeyIsHash); mMetadata = new CacheFileMetadata(mKey); mReady = true; mDataSize = mMetadata->Offset(); @@ -488,7 +477,6 @@ CacheFile::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) this)); mMemoryOnly = true; - MOZ_ASSERT(!mKeyIsHash); mMetadata = new CacheFileMetadata(mKey); mReady = true; mDataSize = mMetadata->Offset(); @@ -530,7 +518,7 @@ CacheFile::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) MOZ_ASSERT(!mMetadata); MOZ_ASSERT(mListener); - mMetadata = new CacheFileMetadata(mHandle, mKey, mKeyIsHash); + mMetadata = new CacheFileMetadata(mHandle, mKey); rv = mMetadata->ReadMetadata(this); if (NS_FAILED(rv)) { @@ -565,13 +553,6 @@ CacheFile::OnMetadataRead(nsresult aResult) bool isNew = false; if (NS_SUCCEEDED(aResult)) { - MOZ_ASSERT(!mMetadata->KeyIsHash()); - - if (mKeyIsHash) { - mMetadata->GetKey(mKey); - mKeyIsHash = false; - } - mReady = true; mDataSize = mMetadata->Offset(); if (mDataSize == 0 && mMetadata->ElementsSize() == 0) { @@ -755,6 +736,10 @@ CacheFile::Doom(CacheFileListener *aCallback) return NS_ERROR_FILE_NOT_FOUND; } + if (mHandle && mHandle->IsDoomed()) { + return NS_ERROR_FILE_NOT_FOUND; + } + nsCOMPtr listener; if (aCallback || !mHandle) { listener = new DoomFileHelper(aCallback); @@ -1384,6 +1369,17 @@ CacheFile::DataSize(int64_t* aSize) return true; } +bool +CacheFile::IsDoomed() +{ + CacheFileAutoLock lock(this); + + if (!mHandle) + return false; + + return mHandle->IsDoomed(); +} + bool CacheFile::IsDirty() { diff --git a/netwerk/cache2/CacheFile.h b/netwerk/cache2/CacheFile.h index 5a459d107d3..e69fda54d51 100644 --- a/netwerk/cache2/CacheFile.h +++ b/netwerk/cache2/CacheFile.h @@ -56,7 +56,6 @@ public: bool aCreateNew, bool aMemoryOnly, bool aPriority, - bool aKeyIsHash, CacheFileListener *aCallback); NS_IMETHOD OnChunkRead(nsresult aResult, CacheFileChunk *aChunk); @@ -98,6 +97,7 @@ public: bool DataSize(int64_t* aSize); void Key(nsACString& aKey) { aKey = mKey; } + bool IsDoomed(); private: friend class CacheFileIOManager; @@ -165,7 +165,6 @@ private: bool mDataAccessed; bool mDataIsDirty; bool mWritingMetadata; - bool mKeyIsHash; nsresult mStatus; int64_t mDataSize; nsCString mKey; diff --git a/netwerk/cache2/CacheFileIOManager.cpp b/netwerk/cache2/CacheFileIOManager.cpp index 5cc5f94fe3f..d95f4a2cc7c 100644 --- a/netwerk/cache2/CacheFileIOManager.cpp +++ b/netwerk/cache2/CacheFileIOManager.cpp @@ -9,8 +9,10 @@ #include "CacheHashUtils.h" #include "CacheStorageService.h" #include "CacheIndex.h" +#include "CacheFileUtils.h" #include "nsThreadUtils.h" #include "CacheFile.h" +#include "CacheObserver.h" #include "nsIFile.h" #include "mozilla/Telemetry.h" #include "mozilla/DebugOnly.h" @@ -36,6 +38,7 @@ namespace net { #define kOpenHandlesLimit 64 #define kMetadataWriteDelay 5000 +#define kEvictionLoopLimit 40 // in milliseconds bool CacheFileHandle::DispatchRelease() @@ -480,32 +483,7 @@ public: if (mTarget) { mRV = NS_OK; - if (mFlags & CacheFileIOManager::SPECIAL_FILE) { - } - else if (mFlags & CacheFileIOManager::NOHASH) { - nsACString::const_char_iterator begin, end; - begin = mKey.BeginReading(); - end = mKey.EndReading(); - uint32_t i = 0; - while (begin != end && i < (SHA1Sum::HashSize << 1)) { - if (!(i & 1)) - mHash[i >> 1] = 0; - uint8_t shift = (i & 1) ? 0 : 4; - if (*begin >= '0' && *begin <= '9') - mHash[i >> 1] |= (*begin - '0') << shift; - else if (*begin >= 'A' && *begin <= 'F') - mHash[i >> 1] |= (*begin - 'A' + 10) << shift; - else - break; - - ++i; - ++begin; - } - - if (i != (SHA1Sum::HashSize << 1) || begin != end) - mRV = NS_ERROR_INVALID_ARG; - } - else { + if (!(mFlags & CacheFileIOManager::SPECIAL_FILE)) { SHA1Sum sum; sum.update(mKey.BeginReading(), mKey.Length()); sum.finish(mHash); @@ -1043,6 +1021,7 @@ NS_IMPL_ISUPPORTS1(CacheFileIOManager, nsITimerCallback) CacheFileIOManager::CacheFileIOManager() : mShuttingDown(false) , mTreeCreated(false) + , mOverLimitEvicting(false) { LOG(("CacheFileIOManager::CacheFileIOManager [this=%p]", this)); MOZ_COUNT_CTOR(CacheFileIOManager); @@ -1781,6 +1760,7 @@ CacheFileIOManager::WriteInternal(CacheFileHandle *aHandle, int64_t aOffset, if (!aHandle->IsDoomed() && !aHandle->IsSpecialFile()) { uint32_t size = aHandle->FileSizeInK(); CacheIndex::UpdateEntry(aHandle->Hash(), nullptr, nullptr, &size); + EvictIfOverLimitInternal(); } } @@ -1861,6 +1841,20 @@ CacheFileIOManager::DoomFileInternal(CacheFileHandle *aHandle) CacheIndex::RemoveEntry(aHandle->Hash()); aHandle->mIsDoomed = true; + + if (!aHandle->IsSpecialFile()) { + nsRefPtr storageService = CacheStorageService::Self(); + if (storageService) { + nsAutoCString url; + nsCOMPtr info; + rv = CacheFileUtils::ParseKey(aHandle->Key(), getter_AddRefs(info), &url); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + if (NS_SUCCEEDED(rv)) { + storageService->CacheFileDoomed(info, url); + } + } + } + return NS_OK; } @@ -2158,6 +2152,181 @@ CacheFileIOManager::RenameFileInternal(CacheFileHandle *aHandle, return NS_OK; } +nsresult +CacheFileIOManager::EvictIfOverLimit() +{ + LOG(("CacheFileIOManager::EvictIfOverLimit()")); + + nsresult rv; + nsRefPtr ioMan = gInstance; + + if (!ioMan) + return NS_ERROR_NOT_INITIALIZED; + + nsCOMPtr ev; + ev = NS_NewRunnableMethod(ioMan, + &CacheFileIOManager::EvictIfOverLimitInternal); + + rv = ioMan->mIOThread->Dispatch(ev, CacheIOThread::EVICT); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +CacheFileIOManager::EvictIfOverLimitInternal() +{ + LOG(("CacheFileIOManager::EvictIfOverLimitInternal()")); + + nsresult rv; + + MOZ_ASSERT(mIOThread->IsCurrentThread()); + + if (mShuttingDown) + return NS_ERROR_NOT_INITIALIZED; + + if (mOverLimitEvicting) { + LOG(("CacheFileIOManager::EvictIfOverLimitInternal() - Eviction already " + "running.")); + return NS_OK; + } + + uint32_t cacheUsage; + rv = CacheIndex::GetCacheSize(&cacheUsage); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t cacheLimit = CacheObserver::DiskCacheCapacity() >> 10; + if (cacheUsage <= cacheLimit) { + LOG(("CacheFileIOManager::EvictIfOverLimitInternal() - Cache size under " + "limit. [cacheSize=%u, limit=%u]", cacheUsage, cacheLimit)); + return NS_OK; + } + + LOG(("CacheFileIOManager::EvictIfOverLimitInternal() - Cache size exceeded " + "limit. Starting overlimit eviction. [cacheSize=%u, limit=%u]", + cacheUsage, cacheLimit)); + + nsCOMPtr ev; + ev = NS_NewRunnableMethod(this, + &CacheFileIOManager::OverLimitEvictionInternal); + + rv = mIOThread->Dispatch(ev, CacheIOThread::EVICT); + NS_ENSURE_SUCCESS(rv, rv); + + mOverLimitEvicting = true; + return NS_OK; +} + +nsresult +CacheFileIOManager::OverLimitEvictionInternal() +{ + LOG(("CacheFileIOManager::OverLimitEvictionInternal()")); + + nsresult rv; + + // mOverLimitEvicting is accessed only on IO thread, so we can set it to false + // here and set ti to true again once we dispatch another event that will + // continue with the eviction. The reason why we do so is that we can fail + // early anywhere in this method and the variable will contain a correct + // value. Otherwise we would need to set it to false on every failing place. + mOverLimitEvicting = false; + + if (mShuttingDown) + return NS_ERROR_NOT_INITIALIZED; + + TimeStamp start; + + while (true) { + uint32_t cacheUsage; + rv = CacheIndex::GetCacheSize(&cacheUsage); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t cacheLimit = CacheObserver::DiskCacheCapacity() >> 10; + if (cacheUsage <= cacheLimit) { + LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Cache size under " + "limit. [cacheSize=%u, limit=%u]", cacheUsage, cacheLimit)); + return NS_OK; + } + + LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Cache size over " + "limit. [cacheSize=%u, limit=%u]", cacheUsage, cacheLimit)); + + if (start.IsNull()) { + start = TimeStamp::NowLoRes(); + } else { + TimeDuration elapsed = TimeStamp::NowLoRes() - start; + if (elapsed.ToMilliseconds() >= kEvictionLoopLimit) { + LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Breaking loop " + "after %u ms.", static_cast(elapsed.ToMilliseconds()))); + break; + } + } + + SHA1Sum::Hash hash; + uint32_t cnt; + static uint32_t consecutiveFailures = 0; + rv = CacheIndex::GetEntryForEviction(&hash, &cnt); + NS_ENSURE_SUCCESS(rv, rv); + + rv = DoomFileByKeyInternal(&hash); + if (NS_SUCCEEDED(rv)) { + consecutiveFailures = 0; + } + else if (rv == NS_ERROR_NOT_AVAILABLE) { + LOG(("CacheFileIOManager::OverLimitEvictionInternal() - " + "DoomFileByKeyInternal() failed. [rv=0x%08x]", rv)); + // TODO index is outdated, start update + + // Make sure index won't return the same entry again + CacheIndex::RemoveEntry(&hash); + consecutiveFailures = 0; + } + else { + // This shouldn't normally happen, but the eviction must not fail + // completely if we ever encounter this problem. + NS_WARNING("CacheFileIOManager::OverLimitEvictionInternal() - Unexpected " + "failure of DoomFileByKeyInternal()"); + + LOG(("CacheFileIOManager::OverLimitEvictionInternal() - " + "DoomFileByKeyInternal() failed. [rv=0x%08x]", rv)); + + // Normally, CacheIndex::UpdateEntry() is called only to update newly + // created/opened entries which are always fresh and UpdateEntry() expects + // and checks this flag. The way we use UpdateEntry() here is a kind of + // hack and we must make sure the flag is set by calling + // EnsureEntryExists(). + rv = CacheIndex::EnsureEntryExists(&hash); + NS_ENSURE_SUCCESS(rv, rv); + + // Move the entry at the end of both lists to make sure we won't end up + // failing on one entry forever. + uint32_t frecency = 0; + uint32_t expTime = nsICacheEntry::NO_EXPIRATION_TIME; + rv = CacheIndex::UpdateEntry(&hash, &frecency, &expTime, nullptr); + NS_ENSURE_SUCCESS(rv, rv); + + consecutiveFailures++; + if (consecutiveFailures >= cnt) { + // This doesn't necessarily mean that we've tried to doom every entry + // but we've reached a sane number of tries. It is likely that another + // eviction will start soon. And as said earlier, this normally doesn't + // happen at all. + return NS_OK; + } + } + } + + nsCOMPtr ev; + ev = NS_NewRunnableMethod(this, + &CacheFileIOManager::OverLimitEvictionInternal); + + rv = mIOThread->Dispatch(ev, CacheIOThread::EVICT); + NS_ENSURE_SUCCESS(rv, rv); + + mOverLimitEvicting = true; + return NS_OK; +} + nsresult CacheFileIOManager::InitIndexEntry(CacheFileHandle *aHandle, uint32_t aAppId, diff --git a/netwerk/cache2/CacheFileIOManager.h b/netwerk/cache2/CacheFileIOManager.h index c468d9b24f7..ab40f907277 100644 --- a/netwerk/cache2/CacheFileIOManager.h +++ b/netwerk/cache2/CacheFileIOManager.h @@ -195,8 +195,7 @@ public: CREATE = 1U, CREATE_NEW = 2U, PRIORITY = 4U, - NOHASH = 8U, - SPECIAL_FILE = 16U + SPECIAL_FILE = 8U }; CacheFileIOManager(); @@ -239,6 +238,8 @@ public: static nsresult RenameFile(CacheFileHandle *aHandle, const nsACString &aNewName, CacheFileIOListener *aCallback); + static nsresult EvictIfOverLimit(); + static nsresult InitIndexEntry(CacheFileHandle *aHandle, uint32_t aAppId, bool aAnonymous, @@ -299,6 +300,8 @@ private: int64_t aTruncatePos, int64_t aEOFPos); nsresult RenameFileInternal(CacheFileHandle *aHandle, const nsACString &aNewName); + nsresult EvictIfOverLimitInternal(); + nsresult OverLimitEvictionInternal(); nsresult CreateFile(CacheFileHandle *aHandle); static void HashToStr(const SHA1Sum::Hash *aHash, nsACString &_retval); @@ -325,6 +328,7 @@ private: nsTArray > mSpecialHandles; nsTArray > mScheduledMetadataWrites; nsCOMPtr mMetadataWritesTimer; + bool mOverLimitEvicting; }; } // net diff --git a/netwerk/cache2/CacheFileMetadata.cpp b/netwerk/cache2/CacheFileMetadata.cpp index e572e7906f8..77b0314aa73 100644 --- a/netwerk/cache2/CacheFileMetadata.cpp +++ b/netwerk/cache2/CacheFileMetadata.cpp @@ -9,6 +9,7 @@ #include "nsICacheEntry.h" #include "CacheHashUtils.h" #include "CacheFileChunk.h" +#include "CacheFileUtils.h" #include "nsILoadContextInfo.h" #include "../cache/nsCacheUtils.h" #include "nsIFile.h" @@ -25,9 +26,8 @@ namespace net { NS_IMPL_ISUPPORTS1(CacheFileMetadata, CacheFileIOListener) -CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString &aKey, bool aKeyIsHash) +CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString &aKey) : mHandle(aHandle) - , mKeyIsHash(aKeyIsHash) , mHashArray(nullptr) , mHashArraySize(0) , mHashCount(0) @@ -48,11 +48,10 @@ CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString memset(&mMetaHdr, 0, sizeof(CacheFileMetadataHeader)); mMetaHdr.mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME; mKey = aKey; - if (!aKeyIsHash) { - DebugOnly rv; - rv = ParseKey(aKey); - MOZ_ASSERT(NS_SUCCEEDED(rv)); - } + + DebugOnly rv; + rv = ParseKey(aKey); + MOZ_ASSERT(NS_SUCCEEDED(rv)); } CacheFileMetadata::~CacheFileMetadata() @@ -79,7 +78,6 @@ CacheFileMetadata::~CacheFileMetadata() CacheFileMetadata::CacheFileMetadata(const nsACString &aKey) : mHandle(nullptr) - , mKeyIsHash(false) , mHashArray(nullptr) , mHashArraySize(0) , mHashCount(0) @@ -110,7 +108,6 @@ CacheFileMetadata::CacheFileMetadata(const nsACString &aKey) CacheFileMetadata::CacheFileMetadata() : mHandle(nullptr) - , mKeyIsHash(false) , mHashArray(nullptr) , mHashArraySize(0) , mHashCount(0) @@ -147,12 +144,6 @@ CacheFileMetadata::GetKey(nsACString &_retval) return NS_OK; } -bool -CacheFileMetadata::KeyIsHash() -{ - return mKeyIsHash; -} - nsresult CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener) { @@ -169,14 +160,6 @@ CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener) MOZ_ASSERT(size != -1); if (size == 0) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::ReadMetadata() - Filesize == 0, cannot create " - "empty metadata since key is a hash. [this=%p]", this)); - - CacheFileIOManager::DoomFile(mHandle, nullptr); - return NS_ERROR_NOT_AVAILABLE; - } - // this is a new entry LOG(("CacheFileMetadata::ReadMetadata() - Filesize == 0, creating empty " "metadata. [this=%p]", this)); @@ -187,15 +170,6 @@ CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener) } if (size < int64_t(sizeof(CacheFileMetadataHeader) + 2*sizeof(uint32_t))) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::ReadMetadata() - File is corrupted, cannot " - "create empty metadata since key is a hash. [this=%p, " - "filesize=%lld]", this, size)); - - CacheFileIOManager::DoomFile(mHandle, nullptr); - return NS_ERROR_FILE_CORRUPTED; - } - // there must be at least checksum, header and offset LOG(("CacheFileMetadata::ReadMetadata() - File is corrupted, creating " "empty metadata. [this=%p, filesize=%lld]", this, size)); @@ -222,15 +196,6 @@ CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener) mListener = aListener; rv = CacheFileIOManager::Read(mHandle, offset, mBuf, mBufSize, this); if (NS_FAILED(rv)) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::ReadMetadata() - CacheFileIOManager::Read() " - "failed synchronously, cannot create empty metadata since key is " - "a hash. [this=%p, rv=0x%08x]", this, rv)); - - CacheFileIOManager::DoomFile(mHandle, nullptr); - return rv; - } - LOG(("CacheFileMetadata::ReadMetadata() - CacheFileIOManager::Read() failed" " synchronously, creating empty metadata. [this=%p, rv=0x%08x]", this, rv)); @@ -253,7 +218,6 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset, MOZ_ASSERT(!mListener); MOZ_ASSERT(!mWriteBuf); - MOZ_ASSERT(!mKeyIsHash); nsresult rv; @@ -377,9 +341,7 @@ CacheFileMetadata::SyncReadMetadata(nsIFile *aFile) return NS_ERROR_FAILURE; } - mKeyIsHash = true; - - rv = ParseMetadata(metaOffset, 0); + rv = ParseMetadata(metaOffset, 0, false); NS_ENSURE_SUCCESS(rv, rv); return NS_OK; @@ -617,21 +579,11 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsCOMPtr listener; if (NS_FAILED(aResult)) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() " - "failed, cannot create empty metadata since key is a hash. [this=%p," - " rv=0x%08x]", this, aResult)); + LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() failed" + ", creating empty metadata. [this=%p, rv=0x%08x]", this, aResult)); - CacheFileIOManager::DoomFile(mHandle, nullptr); - retval = aResult; - } - else { - LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() failed" - ", creating empty metadata. [this=%p, rv=0x%08x]", this, aResult)); - - InitEmptyMetadata(); - retval = NS_OK; - } + InitEmptyMetadata(); + retval = NS_OK; mListener.swap(listener); listener->OnMetadataRead(retval); @@ -646,22 +598,12 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, MOZ_ASSERT(size != -1); if (realOffset >= size) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, cannot create" - "empty metadata since key is a hash. [this=%p, realOffset=%d, " - "size=%lld]", this, realOffset, size)); + LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, creating " + "empty metadata. [this=%p, realOffset=%d, size=%lld]", this, + realOffset, size)); - CacheFileIOManager::DoomFile(mHandle, nullptr); - retval = NS_ERROR_FILE_CORRUPTED; - } - else { - LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, creating " - "empty metadata. [this=%p, realOffset=%d, size=%lld]", this, - realOffset, size)); - - InitEmptyMetadata(); - retval = NS_OK; - } + InitEmptyMetadata(); + retval = NS_OK; mListener.swap(listener); listener->OnMetadataRead(retval); @@ -684,22 +626,12 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, rv = CacheFileIOManager::Read(mHandle, realOffset, mBuf, missing, this); if (NS_FAILED(rv)) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() " - "failed synchronously, cannot create empty metadata since key is " - "a hash. [this=%p, rv=0x%08x]", this, rv)); + LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() " + "failed synchronously, creating empty metadata. [this=%p, " + "rv=0x%08x]", this, rv)); - CacheFileIOManager::DoomFile(mHandle, nullptr); - retval = rv; - } - else { - LOG(("CacheFileMetadata::OnDataRead() - CacheFileIOManager::Read() " - "failed synchronously, creating empty metadata. [this=%p, " - "rv=0x%08x]", this, rv)); - - InitEmptyMetadata(); - retval = NS_OK; - } + InitEmptyMetadata(); + retval = NS_OK; mListener.swap(listener); listener->OnMetadataRead(retval); @@ -711,21 +643,12 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, // We have all data according to offset information at the end of the entry. // Try to parse it. - rv = ParseMetadata(realOffset, realOffset - usedOffset); + rv = ParseMetadata(realOffset, realOffset - usedOffset, true); if (NS_FAILED(rv)) { - if (mKeyIsHash) { - LOG(("CacheFileMetadata::OnDataRead() - Error parsing metadata, cannot " - "create empty metadata since key is a hash. [this=%p]", this)); - - CacheFileIOManager::DoomFile(mHandle, nullptr); - retval = rv; - } - else { - LOG(("CacheFileMetadata::OnDataRead() - Error parsing metadata, creating " - "empty metadata. [this=%p]", this)); - InitEmptyMetadata(); - retval = NS_OK; - } + LOG(("CacheFileMetadata::OnDataRead() - Error parsing metadata, creating " + "empty metadata. [this=%p]", this)); + InitEmptyMetadata(); + retval = NS_OK; } else { retval = NS_OK; @@ -775,10 +698,11 @@ CacheFileMetadata::InitEmptyMetadata() } nsresult -CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset) +CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, + bool aHaveKey) { LOG(("CacheFileMetadata::ParseMetadata() [this=%p, metaOffset=%d, " - "bufOffset=%d]", this, aMetaOffset, aBufOffset)); + "bufOffset=%d, haveKey=%u]", this, aMetaOffset, aBufOffset, aHaveKey)); nsresult rv; @@ -818,16 +742,14 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset) return NS_ERROR_FILE_CORRUPTED; } - nsAutoCString origKey; - uint32_t keySize = reinterpret_cast( mBuf + hdrOffset)->mKeySize; - if (mKeyIsHash) { - // get the original key - origKey.Assign(mBuf + keyOffset, keySize); + if (!aHaveKey) { + // get the key form metadata + mKey.Assign(mBuf + keyOffset, keySize); - rv = ParseKey(origKey); + rv = ParseKey(mKey); if (NS_FAILED(rv)) return rv; } @@ -877,11 +799,6 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset) memmove(mBuf, mBuf + elementsOffset, mElementsSize); mOffset = aMetaOffset; - if (mKeyIsHash) { - mKey = origKey; - mKeyIsHash = false; - } - // TODO: shrink memory if buffer is too big DoMemoryReport(MemoryUsage()); @@ -931,51 +848,15 @@ CacheFileMetadata::EnsureBuffer(uint32_t aSize) nsresult CacheFileMetadata::ParseKey(const nsACString &aKey) { - if (aKey.Length() < 4) { - return NS_ERROR_FAILURE; - } + nsresult rv; - if (aKey[1] == '-') { - mAnonymous = false; - } - else if (aKey[1] == 'A') { - mAnonymous = true; - } - else { - return NS_ERROR_FAILURE; - } + nsCOMPtr info; + rv = CacheFileUtils::ParseKey(aKey, getter_AddRefs(info), nullptr); + NS_ENSURE_SUCCESS(rv, rv); - if (aKey[2] != ':') { - return NS_ERROR_FAILURE; - } - - int32_t appIdEndIdx = aKey.FindChar(':', 3); - if (appIdEndIdx == kNotFound) { - return NS_ERROR_FAILURE; - } - - if (aKey[appIdEndIdx - 1] == 'B') { - mInBrowser = true; - appIdEndIdx--; - } else { - mInBrowser = false; - } - - if (appIdEndIdx < 3) { - return NS_ERROR_FAILURE; - } - - if (appIdEndIdx == 3) { - mAppId = nsILoadContextInfo::NO_APP_ID; - } - else { - nsAutoCString appIdStr(Substring(aKey, 3, appIdEndIdx - 3)); - nsresult rv; - int64_t appId64 = appIdStr.ToInteger64(&rv); - if (NS_FAILED(rv) || appId64 > PR_UINT32_MAX) - return NS_ERROR_FAILURE; - mAppId = appId64; - } + mAnonymous = info->IsAnonymous(); + mAppId = info->AppId(); + mInBrowser = info->IsInBrowserElement(); return NS_OK; } diff --git a/netwerk/cache2/CacheFileMetadata.h b/netwerk/cache2/CacheFileMetadata.h index c1bce22ba55..30fbc4e8998 100644 --- a/netwerk/cache2/CacheFileMetadata.h +++ b/netwerk/cache2/CacheFileMetadata.h @@ -62,15 +62,13 @@ public: NS_DECL_THREADSAFE_ISUPPORTS CacheFileMetadata(CacheFileHandle *aHandle, - const nsACString &aKey, - bool aKeyIsHash); + const nsACString &aKey); CacheFileMetadata(const nsACString &aKey); CacheFileMetadata(); void SetHandle(CacheFileHandle *aHandle); nsresult GetKey(nsACString &_retval); - bool KeyIsHash(); nsresult ReadMetadata(CacheFileMetadataListener *aListener); nsresult WriteMetadata(uint32_t aOffset, @@ -114,14 +112,13 @@ private: virtual ~CacheFileMetadata(); void InitEmptyMetadata(); - nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset); + nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, bool aHaveKey); nsresult CheckElements(const char *aBuf, uint32_t aSize); void EnsureBuffer(uint32_t aSize); nsresult ParseKey(const nsACString &aKey); nsRefPtr mHandle; nsCString mKey; - bool mKeyIsHash; CacheHash::Hash16_t *mHashArray; uint32_t mHashArraySize; uint32_t mHashCount; diff --git a/netwerk/cache2/CacheFileUtils.cpp b/netwerk/cache2/CacheFileUtils.cpp new file mode 100644 index 00000000000..32143d0b361 --- /dev/null +++ b/netwerk/cache2/CacheFileUtils.cpp @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheLog.h" +#include "CacheFileUtils.h" +#include "LoadContextInfo.h" +#include "nsCOMPtr.h" +#include "nsString.h" + + +namespace mozilla { +namespace net { +namespace CacheFileUtils { + +nsresult ParseKey(const nsACString &aKey, + nsILoadContextInfo **aInfo, + nsACString *aURL) +{ + bool isPrivate; + bool isAnonymous; + bool isInBrowser; + uint32_t appId; + + if (aKey.Length() < 4) { + return NS_ERROR_FAILURE; + } + + if (aKey[0] == '-') { + isPrivate = false; + } + else if (aKey[0] == 'P') { + isPrivate = true; + } + else { + return NS_ERROR_FAILURE; + } + + if (aKey[1] == '-') { + isAnonymous = false; + } + else if (aKey[1] == 'A') { + isAnonymous = true; + } + else { + return NS_ERROR_FAILURE; + } + + if (aKey[2] != ':') { + return NS_ERROR_FAILURE; + } + + int32_t appIdEndIdx = aKey.FindChar(':', 3); + if (appIdEndIdx == kNotFound) { + return NS_ERROR_FAILURE; + } + + if (aKey[appIdEndIdx - 1] == 'B') { + isInBrowser = true; + appIdEndIdx--; + } else { + isInBrowser = false; + } + + if (appIdEndIdx < 3) { + return NS_ERROR_FAILURE; + } + + if (appIdEndIdx == 3) { + appId = nsILoadContextInfo::NO_APP_ID; + } + else { + nsAutoCString appIdStr(Substring(aKey, 3, appIdEndIdx - 3)); + nsresult rv; + int64_t appId64 = appIdStr.ToInteger64(&rv); + if (NS_FAILED(rv) || appId64 > PR_UINT32_MAX) + return NS_ERROR_FAILURE; + appId = appId64; + } + + if (aInfo) { + nsCOMPtr info; + info = GetLoadContextInfo(isPrivate, appId, isInBrowser, isAnonymous); + info.forget(aInfo); + } + + if (aURL) { + *aURL = Substring(aKey, appIdEndIdx + 1); + } + + return NS_OK; +} + +void CreateKeyPrefix(nsILoadContextInfo* aInfo, nsACString &_retval) +{ + /** + * This key is used to salt file hashes. When form of the key is changed + * cache entries will fail to find on disk. + */ + _retval.Assign(aInfo->IsPrivate() ? 'P' : '-'); + _retval.Append(aInfo->IsAnonymous() ? 'A' : '-'); + _retval.Append(':'); + if (aInfo->AppId() != nsILoadContextInfo::NO_APP_ID) { + _retval.AppendInt(aInfo->AppId()); + } + if (aInfo->IsInBrowserElement()) { + _retval.Append('B'); + } +} + + +} // CacheFileUtils +} // net +} // mozilla + diff --git a/netwerk/cache2/CacheFileUtils.h b/netwerk/cache2/CacheFileUtils.h new file mode 100644 index 00000000000..f2c9e6e5ee9 --- /dev/null +++ b/netwerk/cache2/CacheFileUtils.h @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef CacheFileUtils__h__ +#define CacheFileUtils__h__ + +#include "nsError.h" + +class nsILoadContextInfo; +class nsACString; + +namespace mozilla { +namespace net { +namespace CacheFileUtils { + +nsresult ParseKey(const nsACString &aKey, + nsILoadContextInfo **aInfo, + nsACString *aURL); + +void CreateKeyPrefix(nsILoadContextInfo* aInfo, nsACString &_retval); + +} // CacheFileUtils +} // net +} // mozilla + +#endif diff --git a/netwerk/cache2/CacheIOThread.h b/netwerk/cache2/CacheIOThread.h index 54f69f53ace..e1521ebad46 100644 --- a/netwerk/cache2/CacheIOThread.h +++ b/netwerk/cache2/CacheIOThread.h @@ -34,8 +34,8 @@ public: WRITE, MANAGEMENT, CLOSE, - EVICT, BUILD_OR_UPDATE_INDEX, + EVICT, LAST_LEVEL }; diff --git a/netwerk/cache2/CacheIndex.cpp b/netwerk/cache2/CacheIndex.cpp index c83b103718e..97b99b5c236 100644 --- a/netwerk/cache2/CacheIndex.cpp +++ b/netwerk/cache2/CacheIndex.cpp @@ -49,6 +49,8 @@ public: , mDoNotSearchInIndex(false) , mDoNotSearchInUpdates(false) { + mIndex->AssertOwnsLock(); + mHash = aHash; CacheIndexEntry *entry = FindEntry(); mIndex->mIndexStats.BeforeChange(entry); @@ -61,6 +63,8 @@ public: ~CacheIndexEntryAutoManage() { + mIndex->AssertOwnsLock(); + CacheIndexEntry *entry = FindEntry(); mIndex->mIndexStats.AfterChange(entry); if (!entry || !entry->IsInitialized() || entry->IsRemoved()) { @@ -81,11 +85,19 @@ public: // record has a different address, we have to replace it replaceFrecency = replaceExpiration = true; } else { - if (entry->mRec->mFrecency != mOldFrecency) { - replaceFrecency = true; + if (entry->mRec->mFrecency == 0 && + entry->mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME) { + // This is a special case when we want to make sure that the entry is + // placed at the end of the lists even when the values didn't change. + replaceFrecency = replaceExpiration = true; } - if (entry->mRec->mExpirationTime != mOldExpirationTime) { - replaceExpiration = true; + else { + if (entry->mRec->mFrecency != mOldFrecency) { + replaceFrecency = true; + } + if (entry->mRec->mExpirationTime != mOldExpirationTime) { + replaceExpiration = true; + } } } @@ -1031,6 +1043,74 @@ CacheIndex::HasEntry(const nsACString &aKey, EntryStatus *_retval) return NS_OK; } +// static +nsresult +CacheIndex::GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt) +{ + LOG(("CacheIndex::GetEntryForEviction()")); + + nsRefPtr index = gInstance; + + if (!index) + return NS_ERROR_NOT_INITIALIZED; + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + MOZ_ASSERT(index->mFrecencyArray.Length() == + index->mExpirationArray.Length()); + + if (index->mExpirationArray.Length() == 0) + return NS_ERROR_NOT_AVAILABLE; + + uint32_t now = PR_Now() / PR_USEC_PER_SEC; + if (index->mExpirationArray[0]->mExpirationTime < now) { + memcpy(aHash, &index->mExpirationArray[0]->mHash, sizeof(SHA1Sum::Hash)); + *aCnt = index->mExpirationArray.Length(); + LOG(("CacheIndex::GetEntryForEviction() - returning entry from expiration " + "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, " + "frecency=%u]", LOGSHA1(aHash), *aCnt, + index->mExpirationArray[0]->mExpirationTime, now, + index->mExpirationArray[0]->mFrecency)); + } + else { + memcpy(aHash, &index->mFrecencyArray[0]->mHash, sizeof(SHA1Sum::Hash)); + *aCnt = index->mFrecencyArray.Length(); + LOG(("CacheIndex::GetEntryForEviction() - returning entry from frecency " + "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, " + "frecency=%u]", LOGSHA1(aHash), *aCnt, + index->mExpirationArray[0]->mExpirationTime, now, + index->mExpirationArray[0]->mFrecency)); + } + + return NS_OK; +} + +// static +nsresult +CacheIndex::GetCacheSize(uint32_t *_retval) +{ + LOG(("CacheIndex::GetCacheSize()")); + + nsRefPtr index = gInstance; + + if (!index) + return NS_ERROR_NOT_INITIALIZED; + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = index->mIndexStats.Size(); + LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval)); + return NS_OK; +} + bool CacheIndex::IsIndexUsable() { @@ -2802,6 +2882,13 @@ CacheIndex::ChangeState(EState aNewState) return; } + // Try to evict entries over limit everytime we're leaving state READING, + // BUILDING or UPDATING, but not during shutdown. + if (!mShuttingDown && aNewState != SHUTDOWN && + (mState == READING || mState == BUILDING || mState == UPDATING)) { + CacheFileIOManager::EvictIfOverLimit(); + } + mState = aNewState; } @@ -2848,6 +2935,13 @@ public: return a->mFrecency == b->mFrecency; } bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const { + // Place entries with frecency 0 at the end of the array. + if (a->mFrecency == 0) { + return false; + } + if (b->mFrecency == 0) { + return true; + } return a->mFrecency < b->mFrecency; } }; diff --git a/netwerk/cache2/CacheIndex.h b/netwerk/cache2/CacheIndex.h index 0916ea585d5..e87f8d073e0 100644 --- a/netwerk/cache2/CacheIndex.h +++ b/netwerk/cache2/CacheIndex.h @@ -348,7 +348,7 @@ public: return mCount - mRemoved - mNotInitialized - mEmpty; } - int64_t Size() { + uint32_t Size() { MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!"); return mSize; } @@ -389,7 +389,7 @@ public: MOZ_ASSERT(mEmpty); mEmpty--; } else { - MOZ_ASSERT(mSize); + MOZ_ASSERT(mSize >= aEntry->GetFileSize()); mSize -= aEntry->GetFileSize(); } } @@ -441,7 +441,7 @@ private: uint32_t mDirty; uint32_t mFresh; uint32_t mEmpty; - int64_t mSize; + uint32_t mSize; #ifdef DEBUG // We completely remove the data about an entry from the stats in // BeforeChange() and set this flag to true. The entry is then modified, @@ -512,6 +512,15 @@ public: // on any thread. static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval); + // Returns a hash of the least important entry that should be evicted if the + // cache size is over limit and also returns a total number of all entries in + // the index. + static nsresult GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt); + + // Returns cache size in kB. + static nsresult GetCacheSize(uint32_t *_retval); + + private: friend class CacheIndexEntryAutoManage; friend class CacheIndexAutoLock; diff --git a/netwerk/cache2/CacheStorageService.cpp b/netwerk/cache2/CacheStorageService.cpp index c2a77a4f47c..4730c9c709a 100644 --- a/netwerk/cache2/CacheStorageService.cpp +++ b/netwerk/cache2/CacheStorageService.cpp @@ -12,6 +12,7 @@ #include "CacheStorage.h" #include "AppCacheStorage.h" #include "CacheEntry.h" +#include "CacheFileUtils.h" #include "OldWrappers.h" #include "nsCacheService.h" @@ -33,23 +34,6 @@ namespace net { namespace { -void LoadContextInfoMappingKey(nsAutoCString &key, nsILoadContextInfo* aInfo) -{ - /** - * This key is used to salt file hashes. When form of the key is changed - * cache entries will fail to find on disk. - */ - key.Append(aInfo->IsPrivate() ? 'P' : '-'); - key.Append(aInfo->IsAnonymous() ? 'A' : '-'); - key.Append(':'); - if (aInfo->AppId() != nsILoadContextInfo::NO_APP_ID) { - key.AppendInt(aInfo->AppId()); - } - if (aInfo->IsInBrowserElement()) { - key.Append('B'); - } -} - void AppendMemoryStorageID(nsAutoCString &key) { key.Append('M'); @@ -513,7 +497,6 @@ public: ~CacheFilesDeletor(); nsresult DeleteAll(); - nsresult DeleteOverLimit(); nsresult DeleteDoomed(); private: @@ -530,7 +513,6 @@ private: uint32_t mRunning; enum { ALL, - OVERLIMIT, DOOMED } mMode; nsresult mRv; @@ -568,12 +550,6 @@ nsresult CacheFilesDeletor::DeleteAll() return Init(CacheFileIOManager::ENTRIES); } -nsresult CacheFilesDeletor::DeleteOverLimit() -{ - mMode = OVERLIMIT; - return Init(CacheFileIOManager::ENTRIES); -} - nsresult CacheFilesDeletor::DeleteDoomed() { mMode = DOOMED; @@ -662,21 +638,6 @@ nsresult CacheFilesDeletor::Execute() TimeStamp start; switch (mMode) { - case OVERLIMIT: - // Examine file by file and delete what is considered expired/unused. - while (mEnumerator->HasMore()) { - rv = mEnumerator->GetNextCacheFile(this); - if (NS_FAILED(rv)) - return rv; - - // Limit up to 5 concurrent file opens - if (mRunning >= 5) - break; - - ++mRunning; - } - break; - case ALL: case DOOMED: // Simply delete all files, don't doom then though the backend @@ -697,13 +658,6 @@ nsresult CacheFilesDeletor::Execute() rv = file->Remove(false); if (NS_FAILED(rv)) { LOG((" could not remove the file, probably doomed, rv=0x%08x", rv)); -#if 0 - // No need to open and doom the file manually since we doom all entries - // we currently have loaded in memory. - rv = mEnumerator->GetCacheFileFromFile(file, this); - if (NS_FAILED(rv)) - return rv; -#endif } ++mRunning; @@ -744,15 +698,6 @@ void CacheFilesDeletor::OnFile(CacheFile* aFile) #endif switch (mMode) { - case OVERLIMIT: - if (mEnumerator->HasMore()) - mEnumerator->GetNextCacheFile(this); - - // NO BREAK ..so far.. - // mayhemer TODO - here we should decide based on frecency and exp time - // whether to delete the file or not. Then we have to check the consumption - // as well. - case ALL: case DOOMED: LOG((" dooming file with key=%s", key.get())); @@ -1208,7 +1153,7 @@ CacheStorageService::AddStorageEntry(CacheStorage const* aStorage, NS_ENSURE_ARG(aStorage); nsAutoCString contextKey; - LoadContextInfoMappingKey(contextKey, aStorage->LoadInfo()); + CacheFileUtils::CreateKeyPrefix(aStorage->LoadInfo(), contextKey); return AddStorageEntry(contextKey, aURI, aIdExtension, aStorage->WriteToDisk(), aCreateIfNotExist, aReplace, @@ -1253,6 +1198,11 @@ CacheStorageService::AddStorageEntry(nsCSubstring const& aContextKey, bool entryExists = entries->Get(entryKey, getter_AddRefs(entry)); + // check whether the file is already doomed + if (entryExists && entry->IsFileDoomed() && !aReplace) { + aReplace = true; + } + // Check entry that is memory-only is also in related memory-only hashtable. // If not, it has been evicted and we will truncate it ; doom is pending for it, // this consumer just made it sooner then the entry has actually been removed @@ -1361,7 +1311,7 @@ CacheStorageService::DoomStorageEntry(CacheStorage const* aStorage, NS_ENSURE_ARG(aURI); nsAutoCString contextKey; - LoadContextInfoMappingKey(contextKey, aStorage->LoadInfo()); + CacheFileUtils::CreateKeyPrefix(aStorage->LoadInfo(), contextKey); nsAutoCString entryKey; nsresult rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey); @@ -1402,7 +1352,7 @@ CacheStorageService::DoomStorageEntry(CacheStorage const* aStorage, if (aStorage->WriteToDisk()) { nsAutoCString contextKey; - LoadContextInfoMappingKey(contextKey, aStorage->LoadInfo()); + CacheFileUtils::CreateKeyPrefix(aStorage->LoadInfo(), contextKey); rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, entryKey); NS_ENSURE_SUCCESS(rv, rv); @@ -1433,7 +1383,7 @@ CacheStorageService::DoomStorageEntries(CacheStorage const* aStorage, NS_ENSURE_ARG(aStorage); nsAutoCString contextKey; - LoadContextInfoMappingKey(contextKey, aStorage->LoadInfo()); + CacheFileUtils::CreateKeyPrefix(aStorage->LoadInfo(), contextKey); mozilla::MutexAutoLock lock(mLock); @@ -1486,12 +1436,39 @@ CacheStorageService::WalkStorageEntries(CacheStorage const* aStorage, NS_ENSURE_ARG(aStorage); nsAutoCString contextKey; - LoadContextInfoMappingKey(contextKey, aStorage->LoadInfo()); + CacheFileUtils::CreateKeyPrefix(aStorage->LoadInfo(), contextKey); nsRefPtr event = new WalkRunnable( contextKey, aVisitEntries, aStorage->WriteToDisk(), aVisitor); return Dispatch(event); } +nsresult +CacheStorageService::CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo, + const nsACString & aURL) +{ + nsRefPtr entry; + nsAutoCString contextKey; + CacheFileUtils::CreateKeyPrefix(aLoadContextInfo, contextKey); + + { + mozilla::MutexAutoLock lock(mLock); + + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + CacheEntryTable* entries; + if (sGlobalEntryTables->Get(contextKey, &entries)) { + entries->Get(aURL, getter_AddRefs(entry)); + } + } + + if (entry && entry->IsFileDoomed()) { + entry->PurgeAndDoom(); + } + + return NS_OK; +} + + } // net } // mozilla diff --git a/netwerk/cache2/CacheStorageService.h b/netwerk/cache2/CacheStorageService.h index 7efbd00d663..649800fa93f 100644 --- a/netwerk/cache2/CacheStorageService.h +++ b/netwerk/cache2/CacheStorageService.h @@ -103,6 +103,7 @@ private: private: // Following methods are thread safe to call. friend class CacheStorage; + friend class CacheFileIOManager; /** * Get, or create when not existing and demanded, an entry for the storage @@ -137,6 +138,14 @@ private: bool aVisitEntries, nsICacheStorageVisitor* aVisitor); + /** + * CacheFileIOManager uses this method to notify CacheStorageService that + * an active entry was removed. This method is called even if the entry + * removal was originated by CacheStorageService. + */ + nsresult CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo, + const nsACString & aURL); + private: friend class CacheMemoryConsumer; diff --git a/netwerk/cache2/moz.build b/netwerk/cache2/moz.build index 80851642c1c..db7f322b3b1 100644 --- a/netwerk/cache2/moz.build +++ b/netwerk/cache2/moz.build @@ -39,6 +39,7 @@ SOURCES += [ 'CacheFileIOManager.cpp', 'CacheFileMetadata.cpp', 'CacheFileOutputStream.cpp', + 'CacheFileUtils.cpp', 'CacheIndex.cpp', 'CacheLog.cpp', 'CacheStorage.cpp',