/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* vim:set ts=4 sw=4 sts=4 cin et: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef _nsDiskCacheMap_h_ #define _nsDiskCacheMap_h_ #include "mozilla/MemoryReporting.h" #include #include "prtypes.h" #include "prnetdb.h" #include "nsDebug.h" #include "nsError.h" #include "nsIFile.h" #include "nsITimer.h" #include "nsDiskCache.h" #include "nsDiskCacheBlockFile.h" class nsDiskCacheBinding; struct nsDiskCacheEntry; /****************************************************************************** * nsDiskCacheRecord * * Cache Location Format * * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit * * 0011 0000 0000 0000 0000 0000 0000 0000 : File Selector (0 = separate file) * 0000 0011 0000 0000 0000 0000 0000 0000 : number of extra contiguous blocks 1-4 * 0100 1100 0000 0000 0000 0000 0000 0000 : reserved bits * 0000 0000 1111 1111 1111 1111 1111 1111 : block# 0-16777216 (2^24) * * 0000 0000 1111 1111 1111 1111 0000 0000 : eFileSizeMask (size of file in k: see note) * 0000 0000 0000 0000 0000 0000 1111 1111 : eFileGenerationMask * * File Selector: * 0 = separate file on disk * 1 = 256 byte block file * 2 = 1k block file * 3 = 4k block file * * eFileSizeMask note: Files larger than 65535 KiB have this limit stored in * the location. The file itself must be examined to * determine its actual size if necessary. * *****************************************************************************/ /* We have 3 block files with roughly the same max size (32MB) 1 - block size 256B, number of blocks 131072 2 - block size 1kB, number of blocks 32768 3 - block size 4kB, number of blocks 8192 */ #define kNumBlockFiles 3 #define SIZE_SHIFT(idx) (2 * ((idx) - 1)) #define BLOCK_SIZE_FOR_INDEX(idx) ((idx) ? (256 << SIZE_SHIFT(idx)) : 0) #define BITMAP_SIZE_FOR_INDEX(idx) ((idx) ? (131072 >> SIZE_SHIFT(idx)) : 0) // Min and max values for the number of records in the DiskCachemap #define kMinRecordCount 512 #define kSeparateFile 0 #define kBuckets (1 << 5) // must be a power of 2! // Maximum size in K which can be stored in the location (see eFileSizeMask). // Both data and metadata can be larger, but only up to kMaxDataSizeK can be // counted into total cache size. I.e. if there are entries where either data or // metadata is larger than kMaxDataSizeK, the total cache size will be // inaccurate (smaller) than the actual cache size. The alternative is to stat // the files to find the real size, which was decided against for performance // reasons. See bug #651100 comment #21. #define kMaxDataSizeK 0xFFFF // preallocate up to 1MB of separate cache file #define kPreallocateLimit 1 * 1024 * 1024 // The minimum amount of milliseconds to wait before re-attempting to // revalidate the cache. #define kRevalidateCacheTimeout 3000 #define kRevalidateCacheTimeoutTolerance 10 #define kRevalidateCacheErrorTimeout 1000 class nsDiskCacheRecord { private: uint32_t mHashNumber; uint32_t mEvictionRank; uint32_t mDataLocation; uint32_t mMetaLocation; enum { eLocationInitializedMask = 0x80000000, eLocationSelectorMask = 0x30000000, eLocationSelectorOffset = 28, eExtraBlocksMask = 0x03000000, eExtraBlocksOffset = 24, eReservedMask = 0x4C000000, eBlockNumberMask = 0x00FFFFFF, eFileSizeMask = 0x00FFFF00, eFileSizeOffset = 8, eFileGenerationMask = 0x000000FF, eFileReservedMask = 0x4F000000 }; public: nsDiskCacheRecord() : mHashNumber(0), mEvictionRank(0), mDataLocation(0), mMetaLocation(0) { } bool ValidRecord() { if ((mDataLocation & eReservedMask) || (mMetaLocation & eReservedMask)) return false; return true; } // HashNumber accessors uint32_t HashNumber() const { return mHashNumber; } void SetHashNumber( uint32_t hashNumber) { mHashNumber = hashNumber; } // EvictionRank accessors uint32_t EvictionRank() const { return mEvictionRank; } void SetEvictionRank( uint32_t rank) { mEvictionRank = rank ? rank : 1; } // DataLocation accessors bool DataLocationInitialized() const { return 0 != (mDataLocation & eLocationInitializedMask); } void ClearDataLocation() { mDataLocation = 0; } uint32_t DataFile() const { return (uint32_t)(mDataLocation & eLocationSelectorMask) >> eLocationSelectorOffset; } void SetDataBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount) { // clear everything mDataLocation = 0; // set file index NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index"); NS_ASSERTION( index > 0,"invalid location index"); mDataLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask; // set startBlock NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number"); mDataLocation |= startBlock & eBlockNumberMask; // set blockCount NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count"); --blockCount; mDataLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask; mDataLocation |= eLocationInitializedMask; } uint32_t DataBlockCount() const { return (uint32_t)((mDataLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1; } uint32_t DataStartBlock() const { return (mDataLocation & eBlockNumberMask); } uint32_t DataBlockSize() const { return BLOCK_SIZE_FOR_INDEX(DataFile()); } uint32_t DataFileSize() const { return (mDataLocation & eFileSizeMask) >> eFileSizeOffset; } void SetDataFileSize(uint32_t size) { NS_ASSERTION((mDataLocation & eFileReservedMask) == 0, "bad location"); mDataLocation &= ~eFileSizeMask; // clear eFileSizeMask mDataLocation |= (size << eFileSizeOffset) & eFileSizeMask; } uint8_t DataFileGeneration() const { return (mDataLocation & eFileGenerationMask); } void SetDataFileGeneration( uint8_t generation) { // clear everything, (separate file index = 0) mDataLocation = 0; mDataLocation |= generation & eFileGenerationMask; mDataLocation |= eLocationInitializedMask; } // MetaLocation accessors bool MetaLocationInitialized() const { return 0 != (mMetaLocation & eLocationInitializedMask); } void ClearMetaLocation() { mMetaLocation = 0; } uint32_t MetaLocation() const { return mMetaLocation; } uint32_t MetaFile() const { return (uint32_t)(mMetaLocation & eLocationSelectorMask) >> eLocationSelectorOffset; } void SetMetaBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount) { // clear everything mMetaLocation = 0; // set file index NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index"); NS_ASSERTION( index > 0, "invalid location index"); mMetaLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask; // set startBlock NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number"); mMetaLocation |= startBlock & eBlockNumberMask; // set blockCount NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count"); --blockCount; mMetaLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask; mMetaLocation |= eLocationInitializedMask; } uint32_t MetaBlockCount() const { return (uint32_t)((mMetaLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1; } uint32_t MetaStartBlock() const { return (mMetaLocation & eBlockNumberMask); } uint32_t MetaBlockSize() const { return BLOCK_SIZE_FOR_INDEX(MetaFile()); } uint32_t MetaFileSize() const { return (mMetaLocation & eFileSizeMask) >> eFileSizeOffset; } void SetMetaFileSize(uint32_t size) { mMetaLocation &= ~eFileSizeMask; // clear eFileSizeMask mMetaLocation |= (size << eFileSizeOffset) & eFileSizeMask; } uint8_t MetaFileGeneration() const { return (mMetaLocation & eFileGenerationMask); } void SetMetaFileGeneration( uint8_t generation) { // clear everything, (separate file index = 0) mMetaLocation = 0; mMetaLocation |= generation & eFileGenerationMask; mMetaLocation |= eLocationInitializedMask; } uint8_t Generation() const { if ((mDataLocation & eLocationInitializedMask) && (DataFile() == 0)) return DataFileGeneration(); if ((mMetaLocation & eLocationInitializedMask) && (MetaFile() == 0)) return MetaFileGeneration(); return 0; // no generation } #if defined(IS_LITTLE_ENDIAN) void Swap() { mHashNumber = htonl(mHashNumber); mEvictionRank = htonl(mEvictionRank); mDataLocation = htonl(mDataLocation); mMetaLocation = htonl(mMetaLocation); } #endif #if defined(IS_LITTLE_ENDIAN) void Unswap() { mHashNumber = ntohl(mHashNumber); mEvictionRank = ntohl(mEvictionRank); mDataLocation = ntohl(mDataLocation); mMetaLocation = ntohl(mMetaLocation); } #endif }; /****************************************************************************** * nsDiskCacheRecordVisitor *****************************************************************************/ enum { kDeleteRecordAndContinue = -1, kStopVisitingRecords = 0, kVisitNextRecord = 1 }; class nsDiskCacheRecordVisitor { public: virtual int32_t VisitRecord( nsDiskCacheRecord * mapRecord) = 0; }; /****************************************************************************** * nsDiskCacheHeader *****************************************************************************/ struct nsDiskCacheHeader { uint32_t mVersion; // cache version. uint32_t mDataSize; // size of cache in units of 1024bytes. int32_t mEntryCount; // number of entries stored in cache. uint32_t mIsDirty; // dirty flag. int32_t mRecordCount; // Number of records uint32_t mEvictionRank[kBuckets]; // Highest EvictionRank of the bucket uint32_t mBucketUsage[kBuckets]; // Number of used entries in the bucket nsDiskCacheHeader() : mVersion(nsDiskCache::kCurrentVersion) , mDataSize(0) , mEntryCount(0) , mIsDirty(true) , mRecordCount(0) {} void Swap() { #if defined(IS_LITTLE_ENDIAN) mVersion = htonl(mVersion); mDataSize = htonl(mDataSize); mEntryCount = htonl(mEntryCount); mIsDirty = htonl(mIsDirty); mRecordCount = htonl(mRecordCount); for (uint32_t i = 0; i < kBuckets ; i++) { mEvictionRank[i] = htonl(mEvictionRank[i]); mBucketUsage[i] = htonl(mBucketUsage[i]); } #endif } void Unswap() { #if defined(IS_LITTLE_ENDIAN) mVersion = ntohl(mVersion); mDataSize = ntohl(mDataSize); mEntryCount = ntohl(mEntryCount); mIsDirty = ntohl(mIsDirty); mRecordCount = ntohl(mRecordCount); for (uint32_t i = 0; i < kBuckets ; i++) { mEvictionRank[i] = ntohl(mEvictionRank[i]); mBucketUsage[i] = ntohl(mBucketUsage[i]); } #endif } }; /****************************************************************************** * nsDiskCacheMap *****************************************************************************/ class nsDiskCacheMap { public: nsDiskCacheMap() : mCacheDirectory(nullptr), mMapFD(nullptr), mCleanFD(nullptr), mRecordArray(nullptr), mBufferSize(0), mBuffer(nullptr), mMaxRecordCount(16384), // this default value won't matter mIsDirtyCacheFlushed(false), mLastInvalidateTime(0) { } ~nsDiskCacheMap() { (void) Close(true); } /** * File Operations * * Open * * Creates a new cache map file if one doesn't exist. * Returns error if it detects change in format or cache wasn't closed. */ nsresult Open( nsIFile * cacheDirectory, nsDiskCache::CorruptCacheInfo * corruptInfo, bool reportCacheCleanTelemetryData); nsresult Close(bool flush); nsresult Trim(); nsresult FlushHeader(); nsresult FlushRecords( bool unswap); void NotifyCapacityChange(uint32_t capacity); /** * Record operations */ nsresult AddRecord( nsDiskCacheRecord * mapRecord, nsDiskCacheRecord * oldRecord); nsresult UpdateRecord( nsDiskCacheRecord * mapRecord); nsresult FindRecord( uint32_t hashNumber, nsDiskCacheRecord * mapRecord); nsresult DeleteRecord( nsDiskCacheRecord * mapRecord); nsresult VisitRecords( nsDiskCacheRecordVisitor * visitor); nsresult EvictRecords( nsDiskCacheRecordVisitor * visitor); /** * Disk Entry operations */ nsresult DeleteStorage( nsDiskCacheRecord * record); nsresult GetFileForDiskCacheRecord( nsDiskCacheRecord * record, bool meta, bool createPath, nsIFile ** result); nsresult GetLocalFileForDiskCacheRecord( nsDiskCacheRecord * record, bool meta, bool createPath, nsIFile ** result); // On success, this returns the buffer owned by nsDiskCacheMap, // so it must not be deleted by the caller. nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord * record); nsresult WriteDiskCacheEntry( nsDiskCacheBinding * binding); nsresult ReadDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size); nsresult WriteDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size); nsresult DeleteStorage( nsDiskCacheRecord * record, bool metaData); /** * Statistical Operations */ void IncrementTotalSize( uint32_t delta) { mHeader.mDataSize += delta; mHeader.mIsDirty = true; } void DecrementTotalSize( uint32_t delta) { NS_ASSERTION(mHeader.mDataSize >= delta, "disk cache size negative?"); mHeader.mDataSize = mHeader.mDataSize > delta ? mHeader.mDataSize - delta : 0; mHeader.mIsDirty = true; } inline void IncrementTotalSize( uint32_t blocks, uint32_t blockSize) { // Round up to nearest K IncrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10); } inline void DecrementTotalSize( uint32_t blocks, uint32_t blockSize) { // Round up to nearest K DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10); } uint32_t TotalSize() { return mHeader.mDataSize; } int32_t EntryCount() { return mHeader.mEntryCount; } size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf); private: /** * Private methods */ nsresult OpenBlockFiles(nsDiskCache::CorruptCacheInfo * corruptInfo); nsresult CloseBlockFiles(bool flush); bool CacheFilesExist(); nsresult CreateCacheSubDirectories(); uint32_t CalculateFileIndex(uint32_t size); nsresult GetBlockFileForIndex( uint32_t index, nsIFile ** result); uint32_t GetBlockSizeForIndex( uint32_t index) const { return BLOCK_SIZE_FOR_INDEX(index); } uint32_t GetBitMapSizeForIndex( uint32_t index) const { return BITMAP_SIZE_FOR_INDEX(index); } // returns the bucket number uint32_t GetBucketIndex( uint32_t hashNumber) const { return (hashNumber & (kBuckets - 1)); } // Gets the size of the bucket (in number of records) uint32_t GetRecordsPerBucket() const { return mHeader.mRecordCount / kBuckets; } // Gets the first record in the bucket nsDiskCacheRecord *GetFirstRecordInBucket(uint32_t bucket) const { return mRecordArray + bucket * GetRecordsPerBucket(); } uint32_t GetBucketRank(uint32_t bucketIndex, uint32_t targetRank); int32_t VisitEachRecord(uint32_t bucketIndex, nsDiskCacheRecordVisitor * visitor, uint32_t evictionRank); nsresult GrowRecords(); nsresult ShrinkRecords(); nsresult EnsureBuffer(uint32_t bufSize); // The returned structure will point to the buffer owned by nsDiskCacheMap, // so it must not be deleted by the caller. nsDiskCacheEntry * CreateDiskCacheEntry(nsDiskCacheBinding * binding, uint32_t * size); // Initializes the _CACHE_CLEAN_ related functionality nsresult InitCacheClean(nsIFile * cacheDirectory, nsDiskCache::CorruptCacheInfo * corruptInfo, bool reportCacheCleanTelemetryData); // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file nsresult WriteCacheClean(bool clean); // Resets the timout for revalidating the cache nsresult ResetCacheTimer(int32_t timeout = kRevalidateCacheTimeout); // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer nsresult InvalidateCache(); // Determines if the cache is in a safe state bool IsCacheInSafeState(); // Revalidates the cache by writting out the header, records, and finally // by calling WriteCacheClean(true). nsresult RevalidateCache(); // Timer which revalidates the cache static void RevalidateTimerCallback(nsITimer *aTimer, void *arg); /** * data members */ private: nsCOMPtr mCleanCacheTimer; nsCOMPtr mCacheDirectory; PRFileDesc * mMapFD; PRFileDesc * mCleanFD; nsDiskCacheRecord * mRecordArray; nsDiskCacheBlockFile mBlockFile[kNumBlockFiles]; uint32_t mBufferSize; char * mBuffer; nsDiskCacheHeader mHeader; int32_t mMaxRecordCount; bool mIsDirtyCacheFlushed; PRIntervalTime mLastInvalidateTime; }; #endif // _nsDiskCacheMap_h_