Bug 597224 - HTTP Cache: use directory tree to store cache files. sr=bzbarsky@mit.edu, a=blocker
authorMichal Novotny <michal.novotny@gmail.com>
Mon, 06 Dec 2010 17:49:05 +0200
changeset 58733 3b2e0fcc3cac9b09a400da839002383de891b088
parent 58732 88fee6b520bbeafc581aa8547fb4f1b131df4a9c
child 58734 b61efde547aa56cafccc1b1fd38ea42d17368799
push id1
push usershaver@mozilla.com
push dateTue, 04 Jan 2011 17:58:04 +0000
reviewersbzbarsky, blocker
bugs597224
milestone2.0b8pre
Bug 597224 - HTTP Cache: use directory tree to store cache files. sr=bzbarsky@mit.edu, a=blocker
netwerk/cache/nsDiskCache.h
netwerk/cache/nsDiskCacheDevice.cpp
netwerk/cache/nsDiskCacheMap.cpp
netwerk/cache/nsDiskCacheMap.h
netwerk/cache/nsDiskCacheStreams.cpp
--- a/netwerk/cache/nsDiskCache.h
+++ b/netwerk/cache/nsDiskCache.h
@@ -48,17 +48,17 @@
 #ifdef XP_WIN
 #include <winsock.h>  // for htonl/ntohl
 #endif
 
 
 class nsDiskCache {
 public:
     enum {
-            kCurrentVersion = 0x00010010      // format = 16 bits major version/16 bits minor version
+            kCurrentVersion = 0x00010011      // format = 16 bits major version/16 bits minor version
     };
 
     enum { kData, kMetaData };
 
     // Parameter initval initializes internal state of hash function. Hash values are different
     // for the same text when different initval is used. It can be any random number.
     // 
     // It can be used for generating 64-bit hash value:
--- a/netwerk/cache/nsDiskCacheDevice.cpp
+++ b/netwerk/cache/nsDiskCacheDevice.cpp
@@ -740,16 +740,17 @@ nsDiskCacheDevice::GetFileForEntry(nsCac
             rv = mCacheMap.UpdateRecord(&binding->mRecord);
             if (NS_FAILED(rv))  return rv;
         }
     }
     
     nsCOMPtr<nsIFile>  file;
     rv = mCacheMap.GetFileForDiskCacheRecord(&binding->mRecord,
                                              nsDiskCache::kData,
+                                             PR_FALSE,
                                              getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = file);
     return NS_OK;
 }
 
 
--- a/netwerk/cache/nsDiskCacheMap.cpp
+++ b/netwerk/cache/nsDiskCacheMap.cpp
@@ -43,16 +43,17 @@
 
 #include "nsDiskCacheMap.h"
 #include "nsDiskCacheBinding.h"
 #include "nsDiskCacheEntry.h"
 
 #include "nsCache.h"
 
 #include <string.h>
+#include "nsPrintfCString.h"
 
 #include "nsISerializable.h"
 #include "nsSerializationHelper.h"
 
 /******************************************************************************
  *  nsDiskCacheMap
  *****************************************************************************/
 
@@ -87,16 +88,19 @@ nsDiskCacheMap::Open(nsILocalFile *  cac
     // check size of map file
     PRUint32 mapSize = PR_Available(mMapFD);    
     if (mapSize == 0) {  // creating a new _CACHE_MAP_
 
         // block files shouldn't exist if we're creating the _CACHE_MAP_
         if (cacheFilesExist)
             goto error_exit;
 
+        if (NS_FAILED(CreateCacheSubDirectories()))
+            goto error_exit;
+
         // create the file - initialize in memory
         memset(&mHeader, 0, sizeof(nsDiskCacheHeader));
         mHeader.mVersion = nsDiskCache::kCurrentVersion;
         mHeader.mRecordCount = kMinRecordCount;
         mRecordArray = (nsDiskCacheRecord *)
             PR_CALLOC(mHeader.mRecordCount * sizeof(nsDiskCacheRecord));
         if (!mRecordArray) {
             rv = NS_ERROR_OUT_OF_MEMORY;
@@ -651,32 +655,61 @@ nsDiskCacheMap::CacheFilesExist()
         rv = blockFile->Exists(&exists);
         if (NS_FAILED(rv) || !exists)  return PR_FALSE;
     }
 
     return PR_TRUE;
 }
 
 
+nsresult
+nsDiskCacheMap::CreateCacheSubDirectories()
+{
+    if (!mCacheDirectory)
+        return NS_ERROR_UNEXPECTED;
+
+    for (PRInt32 index = 0 ; index < 16 ; index++) {
+        nsCOMPtr<nsIFile> file;
+        nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file));
+        if (NS_FAILED(rv))
+            return rv;
+
+        rv = file->AppendNative(nsPrintfCString("%X", index));
+        if (NS_FAILED(rv))
+            return rv;
+
+        nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
+        rv = localFile->Create(nsIFile::DIRECTORY_TYPE, 0700);
+        if (NS_FAILED(rv))
+            return rv;
+    }
+
+    return NS_OK;
+}
+
+
 nsDiskCacheEntry *
 nsDiskCacheMap::ReadDiskCacheEntry(nsDiskCacheRecord * record)
 {
     CACHE_LOG_DEBUG(("CACHE: ReadDiskCacheEntry [%x]\n", record->HashNumber()));
 
     nsresult            rv         = NS_ERROR_UNEXPECTED;
     nsDiskCacheEntry *  diskEntry  = nsnull;
     PRUint32            metaFile   = record->MetaFile();
     PRInt32             bytesRead  = 0;
     
     if (!record->MetaLocationInitialized())  return nsnull;
     
     if (metaFile == 0) {  // entry/metadata stored in separate file
         // open and read the file
         nsCOMPtr<nsILocalFile> file;
-        rv = GetLocalFileForDiskCacheRecord(record, nsDiskCache::kMetaData, getter_AddRefs(file));
+        rv = GetLocalFileForDiskCacheRecord(record,
+                                            nsDiskCache::kMetaData,
+                                            PR_FALSE,
+                                            getter_AddRefs(file));
         NS_ENSURE_SUCCESS(rv, nsnull);
 
         PRFileDesc * fd = nsnull;
         // open the file - restricted to user, the data could be confidential
         rv = file->OpenNSPRFileDesc(PR_RDONLY, 00600, &fd);
         NS_ENSURE_SUCCESS(rv, nsnull);
         
         PRInt32 fileSize = PR_Available(fd);
@@ -815,16 +848,17 @@ nsDiskCacheMap::WriteDiskCacheEntry(nsDi
         // XXX handle metaFileSizeK > USHRT_MAX
         binding->mRecord.SetMetaFileGeneration(binding->mGeneration);
         binding->mRecord.SetMetaFileSize(metaFileSizeK);
         rv = UpdateRecord(&binding->mRecord);
         NS_ENSURE_SUCCESS(rv, rv);
 
         rv = GetLocalFileForDiskCacheRecord(&binding->mRecord,
                                             nsDiskCache::kMetaData,
+                                            PR_TRUE,
                                             getter_AddRefs(localFile));
         NS_ENSURE_SUCCESS(rv, rv);
         
         // open the file
         PRFileDesc * fd;
         // open the file - restricted to user, the data could be confidential
         rv = localFile->OpenNSPRFileDesc(PR_RDWR | PR_TRUNCATE | PR_CREATE_FILE, 00600, &fd);
         NS_ENSURE_SUCCESS(rv, rv);
@@ -939,17 +973,17 @@ nsDiskCacheMap::DeleteStorage(nsDiskCach
     PRUint32    fileIndex = metaData ? record->MetaFile() : record->DataFile();
     nsCOMPtr<nsIFile> file;
     
     if (fileIndex == 0) {
         // delete the file
         PRUint32  sizeK = metaData ? record->MetaFileSize() : record->DataFileSize();
         // XXX if sizeK == USHRT_MAX, stat file for actual size
 
-        rv = GetFileForDiskCacheRecord(record, metaData, getter_AddRefs(file));
+        rv = GetFileForDiskCacheRecord(record, metaData, PR_FALSE, getter_AddRefs(file));
         if (NS_SUCCEEDED(rv)) {
             rv = file->Remove(PR_FALSE);    // false == non-recursive
         }
         DecrementTotalSize(sizeK);
         
     } else if (fileIndex < 4) {
         // deallocate blocks
         PRUint32  startBlock = metaData ? record->MetaStartBlock() : record->DataStartBlock();
@@ -963,42 +997,65 @@ nsDiskCacheMap::DeleteStorage(nsDiskCach
     
     return rv;
 }
 
 
 nsresult
 nsDiskCacheMap::GetFileForDiskCacheRecord(nsDiskCacheRecord * record,
                                           PRBool              meta,
+                                          PRBool              createPath,
                                           nsIFile **          result)
 {
     if (!mCacheDirectory)  return NS_ERROR_NOT_AVAILABLE;
     
     nsCOMPtr<nsIFile> file;
     nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
-    
+
+    PRUint32 hash = record->HashNumber();
+
+    // The file is stored under subdirectories according to the hash number:
+    // 0x01234567 -> 0/12/
+    rv = file->AppendNative(nsPrintfCString("%X", hash >> 28));
+    if (NS_FAILED(rv))  return rv;
+    rv = file->AppendNative(nsPrintfCString("%02X", (hash >> 20) & 0xFF));
+    if (NS_FAILED(rv))  return rv;
+
+    PRBool exists;
+    if (createPath && (NS_FAILED(file->Exists(&exists)) || !exists)) {
+        nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
+        rv = localFile->Create(nsIFile::DIRECTORY_TYPE, 0700);
+        if (NS_FAILED(rv))  return rv;
+    }
+
     PRInt16 generation = record->Generation();
     char name[32];
-    ::sprintf(name, "%08X%c%02X", record->HashNumber(),  (meta ? 'm' : 'd'), generation);
+    // Cut the beginning of the hash that was used in the path
+    ::sprintf(name, "%05X%c%02X", hash & 0xFFFFF, (meta ? 'm' : 'd'),
+              generation);
     rv = file->AppendNative(nsDependentCString(name));
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = file);
     return rv;
 }
 
 
 nsresult
 nsDiskCacheMap::GetLocalFileForDiskCacheRecord(nsDiskCacheRecord * record,
                                                PRBool              meta,
+                                               PRBool              createPath,
                                                nsILocalFile **     result)
 {
     nsCOMPtr<nsIFile> file;
-    nsresult rv = GetFileForDiskCacheRecord(record, meta, getter_AddRefs(file));
+    nsresult rv = GetFileForDiskCacheRecord(record,
+                                            meta,
+                                            createPath,
+                                            getter_AddRefs(file));
     if (NS_FAILED(rv))  return rv;
     
     nsCOMPtr<nsILocalFile> localFile = do_QueryInterface(file, &rv);
     if (NS_FAILED(rv))  return rv;
     
     NS_IF_ADDREF(*result = localFile);
     return rv;
 }
--- a/netwerk/cache/nsDiskCacheMap.h
+++ b/netwerk/cache/nsDiskCacheMap.h
@@ -436,20 +436,22 @@ public:
 
 /**
  *  Disk Entry operations
  */
     nsresult    DeleteStorage( nsDiskCacheRecord *  record);
 
     nsresult    GetFileForDiskCacheRecord( nsDiskCacheRecord * record,
                                            PRBool              meta,
+                                           PRBool              createPath,
                                            nsIFile **          result);
                                           
     nsresult    GetLocalFileForDiskCacheRecord( nsDiskCacheRecord *  record,
                                                 PRBool               meta,
+                                                PRBool               createPath,
                                                 nsILocalFile **      result);
 
     // On success, this returns the buffer owned by nsDiskCacheMap,
     // so it must not be deleted by the caller.
     nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord *  record);
 
     nsresult    WriteDiskCacheEntry( nsDiskCacheBinding *  binding);
     
@@ -494,16 +496,18 @@ private:
 
     /**
      *  Private methods
      */
     nsresult    OpenBlockFiles();
     nsresult    CloseBlockFiles(PRBool flush);
     PRBool      CacheFilesExist();
 
+    nsresult    CreateCacheSubDirectories();
+
     PRUint32    CalculateFileIndex(PRUint32 size);
 
     nsresult    GetBlockFileForIndex( PRUint32 index, nsILocalFile ** result);
     PRUint32    GetBlockSizeForIndex( PRUint32 index) const {
         return BLOCK_SIZE_FOR_INDEX(index);
     }
     
     // returns the bucket number    
--- a/netwerk/cache/nsDiskCacheStreams.cpp
+++ b/netwerk/cache/nsDiskCacheStreams.cpp
@@ -663,16 +663,17 @@ nsDiskCacheStreamIO::OpenCacheFile(PRInt
 {
     NS_ENSURE_ARG_POINTER(fd);
     
     nsresult         rv;
     nsDiskCacheMap * cacheMap = mDevice->CacheMap();
     
     rv = cacheMap->GetLocalFileForDiskCacheRecord(&mBinding->mRecord,
                                                   nsDiskCache::kData,
+                                                  !!(flags & PR_CREATE_FILE),
                                                   getter_AddRefs(mLocalFile));
     if (NS_FAILED(rv))  return rv;
     
     // create PRFileDesc for input stream - the 00600 is just for consistency
     rv = mLocalFile->OpenNSPRFileDesc(flags, 00600, fd);
     if (NS_FAILED(rv))  return rv;  // unable to open file
 
     return NS_OK;