Bug 597224 - HTTP Cache: use directory tree to store cache files. sr=bzbarsky@mit.edu, a=blocker
authorMichal Novotny <michal.novotny@gmail.com>
Mon, 06 Dec 2010 17:49:05 +0200
changeset 58733 3b2e0fcc3cac9b09a400da839002383de891b088
parent 58732 88fee6b520bbeafc581aa8547fb4f1b131df4a9c
child 58734 b61efde547aa56cafccc1b1fd38ea42d17368799
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-beta@bfdb6e623a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbzbarsky, blocker
bugs597224
milestone2.0b8pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 597224 - HTTP Cache: use directory tree to store cache files. sr=bzbarsky@mit.edu, a=blocker
netwerk/cache/nsDiskCache.h
netwerk/cache/nsDiskCacheDevice.cpp
netwerk/cache/nsDiskCacheMap.cpp
netwerk/cache/nsDiskCacheMap.h
netwerk/cache/nsDiskCacheStreams.cpp
--- a/netwerk/cache/nsDiskCache.h
+++ b/netwerk/cache/nsDiskCache.h
@@ -48,17 +48,17 @@
 #ifdef XP_WIN
 #include <winsock.h>  // for htonl/ntohl
 #endif
 
 
 class nsDiskCache {
 public:
     enum {
-            kCurrentVersion = 0x00010010      // format = 16 bits major version/16 bits minor version
+            kCurrentVersion = 0x00010011      // format = 16 bits major version/16 bits minor version
     };
 
     enum { kData, kMetaData };
 
     // Parameter initval initializes internal state of hash function. Hash values are different
     // for the same text when different initval is used. It can be any random number.
     // 
     // It can be used for generating 64-bit hash value:
--- a/netwerk/cache/nsDiskCacheDevice.cpp
+++ b/netwerk/cache/nsDiskCacheDevice.cpp
@@ -740,16 +740,17 @@ nsDiskCacheDevice::GetFileForEntry(nsCac
             rv = mCacheMap.UpdateRecord(&binding->mRecord);
             if (NS_FAILED(rv))  return rv;
         }
     }
     
     nsCOMPtr<nsIFile>  file;
     rv = mCacheMap.GetFileForDiskCacheRecord(&binding->mRecord,
                                              nsDiskCache::kData,
+                                             PR_FALSE,
                                              getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = file);
     return NS_OK;
 }
 
 
--- a/netwerk/cache/nsDiskCacheMap.cpp
+++ b/netwerk/cache/nsDiskCacheMap.cpp
@@ -43,16 +43,17 @@
 
 #include "nsDiskCacheMap.h"
 #include "nsDiskCacheBinding.h"
 #include "nsDiskCacheEntry.h"
 
 #include "nsCache.h"
 
 #include <string.h>
+#include "nsPrintfCString.h"
 
 #include "nsISerializable.h"
 #include "nsSerializationHelper.h"
 
 /******************************************************************************
  *  nsDiskCacheMap
  *****************************************************************************/
 
@@ -87,16 +88,19 @@ nsDiskCacheMap::Open(nsILocalFile *  cac
     // check size of map file
     PRUint32 mapSize = PR_Available(mMapFD);    
     if (mapSize == 0) {  // creating a new _CACHE_MAP_
 
         // block files shouldn't exist if we're creating the _CACHE_MAP_
         if (cacheFilesExist)
             goto error_exit;
 
+        if (NS_FAILED(CreateCacheSubDirectories()))
+            goto error_exit;
+
         // create the file - initialize in memory
         memset(&mHeader, 0, sizeof(nsDiskCacheHeader));
         mHeader.mVersion = nsDiskCache::kCurrentVersion;
         mHeader.mRecordCount = kMinRecordCount;
         mRecordArray = (nsDiskCacheRecord *)
             PR_CALLOC(mHeader.mRecordCount * sizeof(nsDiskCacheRecord));
         if (!mRecordArray) {
             rv = NS_ERROR_OUT_OF_MEMORY;
@@ -651,32 +655,61 @@ nsDiskCacheMap::CacheFilesExist()
         rv = blockFile->Exists(&exists);
         if (NS_FAILED(rv) || !exists)  return PR_FALSE;
     }
 
     return PR_TRUE;
 }
 
 
+nsresult
+nsDiskCacheMap::CreateCacheSubDirectories()
+{
+    if (!mCacheDirectory)
+        return NS_ERROR_UNEXPECTED;
+
+    for (PRInt32 index = 0 ; index < 16 ; index++) {
+        nsCOMPtr<nsIFile> file;
+        nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file));
+        if (NS_FAILED(rv))
+            return rv;
+
+        rv = file->AppendNative(nsPrintfCString("%X", index));
+        if (NS_FAILED(rv))
+            return rv;
+
+        nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
+        rv = localFile->Create(nsIFile::DIRECTORY_TYPE, 0700);
+        if (NS_FAILED(rv))
+            return rv;
+    }
+
+    return NS_OK;
+}
+
+
 nsDiskCacheEntry *
 nsDiskCacheMap::ReadDiskCacheEntry(nsDiskCacheRecord * record)
 {
     CACHE_LOG_DEBUG(("CACHE: ReadDiskCacheEntry [%x]\n", record->HashNumber()));
 
     nsresult            rv         = NS_ERROR_UNEXPECTED;
     nsDiskCacheEntry *  diskEntry  = nsnull;
     PRUint32            metaFile   = record->MetaFile();
     PRInt32             bytesRead  = 0;
     
     if (!record->MetaLocationInitialized())  return nsnull;
     
     if (metaFile == 0) {  // entry/metadata stored in separate file
         // open and read the file
         nsCOMPtr<nsILocalFile> file;
-        rv = GetLocalFileForDiskCacheRecord(record, nsDiskCache::kMetaData, getter_AddRefs(file));
+        rv = GetLocalFileForDiskCacheRecord(record,
+                                            nsDiskCache::kMetaData,
+                                            PR_FALSE,
+                                            getter_AddRefs(file));
         NS_ENSURE_SUCCESS(rv, nsnull);
 
         PRFileDesc * fd = nsnull;
         // open the file - restricted to user, the data could be confidential
         rv = file->OpenNSPRFileDesc(PR_RDONLY, 00600, &fd);
         NS_ENSURE_SUCCESS(rv, nsnull);
         
         PRInt32 fileSize = PR_Available(fd);
@@ -815,16 +848,17 @@ nsDiskCacheMap::WriteDiskCacheEntry(nsDi
         // XXX handle metaFileSizeK > USHRT_MAX
         binding->mRecord.SetMetaFileGeneration(binding->mGeneration);
         binding->mRecord.SetMetaFileSize(metaFileSizeK);
         rv = UpdateRecord(&binding->mRecord);
         NS_ENSURE_SUCCESS(rv, rv);
 
         rv = GetLocalFileForDiskCacheRecord(&binding->mRecord,
                                             nsDiskCache::kMetaData,
+                                            PR_TRUE,
                                             getter_AddRefs(localFile));
         NS_ENSURE_SUCCESS(rv, rv);
         
         // open the file
         PRFileDesc * fd;
         // open the file - restricted to user, the data could be confidential
         rv = localFile->OpenNSPRFileDesc(PR_RDWR | PR_TRUNCATE | PR_CREATE_FILE, 00600, &fd);
         NS_ENSURE_SUCCESS(rv, rv);
@@ -939,17 +973,17 @@ nsDiskCacheMap::DeleteStorage(nsDiskCach
     PRUint32    fileIndex = metaData ? record->MetaFile() : record->DataFile();
     nsCOMPtr<nsIFile> file;
     
     if (fileIndex == 0) {
         // delete the file
         PRUint32  sizeK = metaData ? record->MetaFileSize() : record->DataFileSize();
         // XXX if sizeK == USHRT_MAX, stat file for actual size
 
-        rv = GetFileForDiskCacheRecord(record, metaData, getter_AddRefs(file));
+        rv = GetFileForDiskCacheRecord(record, metaData, PR_FALSE, getter_AddRefs(file));
         if (NS_SUCCEEDED(rv)) {
             rv = file->Remove(PR_FALSE);    // false == non-recursive
         }
         DecrementTotalSize(sizeK);
         
     } else if (fileIndex < 4) {
         // deallocate blocks
         PRUint32  startBlock = metaData ? record->MetaStartBlock() : record->DataStartBlock();
@@ -963,42 +997,65 @@ nsDiskCacheMap::DeleteStorage(nsDiskCach
     
     return rv;
 }
 
 
 nsresult
 nsDiskCacheMap::GetFileForDiskCacheRecord(nsDiskCacheRecord * record,
                                           PRBool              meta,
+                                          PRBool              createPath,
                                           nsIFile **          result)
 {
     if (!mCacheDirectory)  return NS_ERROR_NOT_AVAILABLE;
     
     nsCOMPtr<nsIFile> file;
     nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
-    
+
+    PRUint32 hash = record->HashNumber();
+
+    // The file is stored under subdirectories according to the hash number:
+    // 0x01234567 -> 0/12/
+    rv = file->AppendNative(nsPrintfCString("%X", hash >> 28));
+    if (NS_FAILED(rv))  return rv;
+    rv = file->AppendNative(nsPrintfCString("%02X", (hash >> 20) & 0xFF));
+    if (NS_FAILED(rv))  return rv;
+
+    PRBool exists;
+    if (createPath && (NS_FAILED(file->Exists(&exists)) || !exists)) {
+        nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
+        rv = localFile->Create(nsIFile::DIRECTORY_TYPE, 0700);
+        if (NS_FAILED(rv))  return rv;
+    }
+
     PRInt16 generation = record->Generation();
     char name[32];
-    ::sprintf(name, "%08X%c%02X", record->HashNumber(),  (meta ? 'm' : 'd'), generation);
+    // Cut the beginning of the hash that was used in the path
+    ::sprintf(name, "%05X%c%02X", hash & 0xFFFFF, (meta ? 'm' : 'd'),
+              generation);
     rv = file->AppendNative(nsDependentCString(name));
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = file);
     return rv;
 }
 
 
 nsresult
 nsDiskCacheMap::GetLocalFileForDiskCacheRecord(nsDiskCacheRecord * record,
                                                PRBool              meta,
+                                               PRBool              createPath,
                                                nsILocalFile **     result)
 {
     nsCOMPtr<nsIFile> file;
-    nsresult rv = GetFileForDiskCacheRecord(record, meta, getter_AddRefs(file));
+    nsresult rv = GetFileForDiskCacheRecord(record,
+                                            meta,
+                                            createPath,
+                                            getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
     
     nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = localFile);
     return rv;
 }
--- a/netwerk/cache/nsDiskCacheMap.h
+++ b/netwerk/cache/nsDiskCacheMap.h
@@ -436,20 +436,22 @@ public:
 
 /**
  *  Disk Entry operations
  */
     nsresult    DeleteStorage( nsDiskCacheRecord *  record);
 
     nsresult    GetFileForDiskCacheRecord( nsDiskCacheRecord * record,
                                            PRBool              meta,
+                                           PRBool              createPath,
                                            nsIFile **          result);
                                           
     nsresult    GetLocalFileForDiskCacheRecord( nsDiskCacheRecord *  record,
                                                 PRBool               meta,
+                                                PRBool               createPath,
                                                 nsILocalFile **      result);
 
     // On success, this returns the buffer owned by nsDiskCacheMap,
     // so it must not be deleted by the caller.
     nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord *  record);
 
     nsresult    WriteDiskCacheEntry( nsDiskCacheBinding *  binding);
     
@@ -494,16 +496,18 @@ private:
 
     /**
      *  Private methods
      */
     nsresult    OpenBlockFiles();
     nsresult    CloseBlockFiles(PRBool flush);
     PRBool      CacheFilesExist();
 
+    nsresult    CreateCacheSubDirectories();
+
     PRUint32    CalculateFileIndex(PRUint32 size);
 
     nsresult    GetBlockFileForIndex( PRUint32 index, nsILocalFile ** result);
     PRUint32    GetBlockSizeForIndex( PRUint32 index) const {
         return BLOCK_SIZE_FOR_INDEX(index);
     }
     
     // returns the bucket number    
--- a/netwerk/cache/nsDiskCacheStreams.cpp
+++ b/netwerk/cache/nsDiskCacheStreams.cpp
@@ -663,16 +663,17 @@ nsDiskCacheStreamIO::OpenCacheFile(PRInt
 {
     NS_ENSURE_ARG_POINTER(fd);
     
     nsresult         rv;
     nsDiskCacheMap * cacheMap = mDevice->CacheMap();
     
     rv = cacheMap->GetLocalFileForDiskCacheRecord(&mBinding->mRecord,
                                                   nsDiskCache::kData,
+                                                  !!(flags & PR_CREATE_FILE),
                                                   getter_AddRefs(mLocalFile));
     if (NS_FAILED(rv))  return rv;
     
     // create PRFileDesc for input stream - the 00600 is just for consistency
     rv = mLocalFile->OpenNSPRFileDesc(flags, 00600, fd);
     if (NS_FAILED(rv))  return rv;  // unable to open file
 
     return NS_OK;