Bug 754671 - [Page Thumbnails] size of thumbnails directory (in profiles directory) keeps growing infinitely; r=felipe
authorTim Taubert <tim.taubert@gmx.de>
Tue, 21 Aug 2012 11:17:18 -0700
changeset 105041 327883b4f2fe04f20f7ab0d1f0cd3603d85307d2
parent 104890 688a80cdf39fc0c26b4558702800a6cc279ddea0
child 105042 8e30c456cb6e268748013d1dc81f8e1dac3e2fd8
push id55
push usershu@rfrn.org
push dateThu, 30 Aug 2012 01:33:09 +0000
reviewersfelipe
bugs754671
milestone17.0a1
Bug 754671 - [Page Thumbnails] size of thumbnails directory (in profiles directory) keeps growing infinitely; r=felipe
browser/base/content/browser-thumbnails.js
browser/components/nsBrowserGlue.js
browser/components/thumbnails/Makefile.in
browser/components/thumbnails/PageThumbs.jsm
browser/components/thumbnails/PageThumbsWorker.js
browser/components/thumbnails/test/Makefile.in
browser/components/thumbnails/test/browser_thumbnails_bug753755.js
browser/modules/NewTabUtils.jsm
--- a/browser/base/content/browser-thumbnails.js
+++ b/browser/base/content/browser-thumbnails.js
@@ -31,30 +31,32 @@ let gBrowserThumbnails = {
   _tabEvents: ["TabClose", "TabSelect"],
 
   init: function Thumbnails_init() {
     try {
       if (Services.prefs.getBoolPref("browser.pagethumbnails.capturing_disabled"))
         return;
     } catch (e) {}
 
+    PageThumbs.addExpirationFilter(this);
     gBrowser.addTabsProgressListener(this);
     Services.prefs.addObserver(this.PREF_DISK_CACHE_SSL, this, false);
 
     this._sslDiskCacheEnabled =
       Services.prefs.getBoolPref(this.PREF_DISK_CACHE_SSL);
 
     this._tabEvents.forEach(function (aEvent) {
       gBrowser.tabContainer.addEventListener(aEvent, this, false);
     }, this);
 
     this._timeouts = new WeakMap();
   },
 
   uninit: function Thumbnails_uninit() {
+    PageThumbs.removeExpirationFilter(this);
     gBrowser.removeTabsProgressListener(this);
     Services.prefs.removeObserver(this.PREF_DISK_CACHE_SSL, this);
 
     this._tabEvents.forEach(function (aEvent) {
       gBrowser.tabContainer.removeEventListener(aEvent, this, false);
     }, this);
   },
 
@@ -75,16 +77,21 @@ let gBrowserThumbnails = {
     }
   },
 
   observe: function Thumbnails_observe() {
     this._sslDiskCacheEnabled =
       Services.prefs.getBoolPref(this.PREF_DISK_CACHE_SSL);
   },
 
+  filterForThumbnailExpiration:
+  function Thumbnails_filterForThumbnailExpiration(aCallback) {
+    aCallback([browser.currentURI.spec for (browser of gBrowser.browsers)]);
+  },
+
   /**
    * State change progress listener for all tabs.
    */
   onStateChange: function Thumbnails_onStateChange(aBrowser, aWebProgress,
                                                    aRequest, aStateFlags, aStatus) {
     if (aStateFlags & Ci.nsIWebProgressListener.STATE_STOP &&
         aStateFlags & Ci.nsIWebProgressListener.STATE_IS_NETWORK)
       this._delayedCapture(aBrowser);
--- a/browser/components/nsBrowserGlue.js
+++ b/browser/components/nsBrowserGlue.js
@@ -27,16 +27,19 @@ XPCOMUtils.defineLazyModuleGetter(this, 
                                   "resource://gre/modules/BookmarkHTMLUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "webappsUI",
                                   "resource:///modules/webappsUI.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
                                   "resource:///modules/PageThumbs.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils",
+                                  "resource:///modules/NewTabUtils.jsm");
+
 XPCOMUtils.defineLazyModuleGetter(this, "PdfJs",
                                   "resource://pdf.js/PdfJs.jsm");
 
 const PREF_PLUGINS_NOTIFYUSER = "plugins.update.notifyUser";
 const PREF_PLUGINS_UPDATEURL  = "plugins.update.url";
 
 // We try to backup bookmarks at idle times, to avoid doing that at shutdown.
 // Number of idle seconds before trying to backup bookmarks.  15 minutes.
@@ -333,16 +336,17 @@ BrowserGlue.prototype = {
 
     // handle any UI migration
     this._migrateUI();
 
     // Initialize webapps UI
     webappsUI.init();
 
     PageThumbs.init();
+    NewTabUtils.init();
 
     SignInToWebsiteUX.init();
 
     PdfJs.init();
 
     Services.obs.notifyObservers(null, "browser-ui-startup-complete", "");
   },
 
--- a/browser/components/thumbnails/Makefile.in
+++ b/browser/components/thumbnails/Makefile.in
@@ -10,16 +10,17 @@ VPATH		= @srcdir@
 include $(DEPTH)/config/autoconf.mk
 
 EXTRA_COMPONENTS = \
 	BrowserPageThumbs.manifest \
 	PageThumbsProtocol.js \
 	$(NULL)
 
 EXTRA_JS_MODULES = \
+	PageThumbsWorker.js \
 	PageThumbs.jsm \
 	$(NULL)
 
 TEST_DIRS += test
 
 include $(topsrcdir)/config/rules.mk
 
 XPIDL_FLAGS += -I$(topsrcdir)/browser/components/
--- a/browser/components/thumbnails/PageThumbs.jsm
+++ b/browser/components/thumbnails/PageThumbs.jsm
@@ -7,17 +7,20 @@
 let EXPORTED_SYMBOLS = ["PageThumbs", "PageThumbsStorage", "PageThumbsCache"];
 
 const Cu = Components.utils;
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 
 const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
 const PREF_STORAGE_VERSION = "browser.pagethumbnails.storage_version";
-const LATEST_STORAGE_VERSION = 1;
+const LATEST_STORAGE_VERSION = 2;
+
+const EXPIRATION_MIN_CHUNK_SIZE = 50;
+const EXPIRATION_INTERVAL_SECS = 3600;
 
 /**
  * Name of the directory in the profile that contains the thumbnails.
  */
 const THUMBNAIL_DIRECTORY = "thumbnails";
 
 /**
  * The default background color for page thumbnails.
@@ -33,16 +36,19 @@ XPCOMUtils.defineLazyModuleGetter(this, 
   "resource://gre/modules/Services.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
   "resource://gre/modules/FileUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
   "resource://gre/modules/PlacesUtils.jsm");
 
+XPCOMUtils.defineLazyServiceGetter(this, "gUpdateTimerManager",
+  "@mozilla.org/updates/timer-manager;1", "nsIUpdateTimerManager");
+
 XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function () {
   return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
 });
 
 XPCOMUtils.defineLazyGetter(this, "gUnicodeConverter", function () {
   let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
                     .createInstance(Ci.nsIScriptableUnicodeConverter);
   converter.charset = 'utf8';
@@ -79,16 +85,17 @@ let PageThumbs = {
 
   init: function PageThumbs_init() {
     if (!this._initialized) {
       this._initialized = true;
       PlacesUtils.history.addObserver(PageThumbsHistoryObserver, false);
 
       // Migrate the underlying storage, if needed.
       PageThumbsStorageMigrator.migrate();
+      PageThumbsExpiration.init();
     }
   },
 
   uninit: function PageThumbs_uninit() {
     if (this._initialized) {
       this._initialized = false;
       PlacesUtils.history.removeObserver(PageThumbsHistoryObserver);
     }
@@ -193,16 +200,24 @@ let PageThumbs = {
         if (aCallback)
           aCallback(aSuccessful);
       }
 
       PageThumbsStorage.write(url, aInputStream, finish);
     });
   },
 
+  addExpirationFilter: function PageThumbs_addExpirationFilter(aFilter) {
+    PageThumbsExpiration.addFilter(aFilter);
+  },
+
+  removeExpirationFilter: function PageThumbs_removeExpirationFilter(aFilter) {
+    PageThumbsExpiration.removeFilter(aFilter);
+  },
+
   /**
    * Determines the crop size for a given content window.
    * @param aWindow The content window.
    * @param aCanvas The target canvas.
    * @return An array containing width, height and scale.
    */
   _determineCropSize: function PageThumbs_determineCropSize(aWindow, aCanvas) {
     let sw = aWindow.innerWidth;
@@ -259,26 +274,33 @@ let PageThumbs = {
     }
     catch (e) {
       return true;
     }
   },
 };
 
 let PageThumbsStorage = {
-  getFileForURL: function Storage_getFileForURL(aURL, aOptions) {
+  getDirectory: function Storage_getDirectory(aCreate = true) {
+    return FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY], aCreate);
+  },
+
+  getLeafNameForURL: function Storage_getLeafNameForURL(aURL) {
     let hash = this._calculateMD5Hash(aURL);
-    let parts = [THUMBNAIL_DIRECTORY, hash[0], hash[1]];
-    let file = FileUtils.getDir("ProfLD", parts, aOptions && aOptions.createPath);
-    file.append(hash.slice(2) + ".png");
+    return hash + ".png";
+  },
+
+  getFileForURL: function Storage_getFileForURL(aURL) {
+    let file = this.getDirectory();
+    file.append(this.getLeafNameForURL(aURL));
     return file;
   },
 
   write: function Storage_write(aURL, aDataStream, aCallback) {
-    let file = this.getFileForURL(aURL, {createPath: true});
+    let file = this.getFileForURL(aURL);
     let fos = FileUtils.openSafeFileOutputStream(file);
 
     NetUtil.asyncCopy(aDataStream, fos, function (aResult) {
       FileUtils.closeSafeFileOutputStream(fos);
       aCallback(Components.isSuccessCode(aResult));
     });
   },
 
@@ -289,28 +311,27 @@ let PageThumbsStorage = {
     try {
       sourceFile.copyTo(targetFile.parent, targetFile.leafName);
     } catch (e) {
       /* We might not be permitted to write to the file. */
     }
   },
 
   remove: function Storage_remove(aURL) {
-    try {
-      this.getFileForURL(aURL).remove(false);
-    } catch (e) {
-      /* The file might not exist or we're not permitted to remove it. */
-    }
+    let file = this.getFileForURL(aURL);
+    PageThumbsWorker.postMessage({type: "removeFile", path: file.path});
   },
 
   wipe: function Storage_wipe() {
+    let dir = this.getDirectory(false);
+    dir.followLinks = false;
     try {
-      FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY]).remove(true);
+      dir.remove(true);
     } catch (e) {
-      /* The file might not exist or we're not permitted to remove it. */
+      /* The directory might not exist or we're not permitted to remove it. */
     }
   },
 
   _calculateMD5Hash: function Storage_calculateMD5Hash(aValue) {
     let hash = gCryptoHash;
     let value = gUnicodeConverter.convertToByteArray(aValue);
 
     hash.init(hash.MD5);
@@ -318,18 +339,17 @@ let PageThumbsStorage = {
     return this._convertToHexString(hash.finish(false));
   },
 
   _convertToHexString: function Storage_convertToHexString(aData) {
     let hex = "";
     for (let i = 0; i < aData.length; i++)
       hex += ("0" + aData.charCodeAt(i).toString(16)).slice(-2);
     return hex;
-  },
-
+  }
 };
 
 let PageThumbsStorageMigrator = {
   get currentVersion() {
     try {
       return Services.prefs.getIntPref(PREF_STORAGE_VERSION);
     } catch (e) {
       // The pref doesn't exist, yet. Return version 0.
@@ -339,34 +359,164 @@ let PageThumbsStorageMigrator = {
 
   set currentVersion(aVersion) {
     Services.prefs.setIntPref(PREF_STORAGE_VERSION, aVersion);
   },
 
   migrate: function Migrator_migrate() {
     let version = this.currentVersion;
 
-    if (version < 1)
+    if (version < 1) {
       this.removeThumbnailsFromRoamingProfile();
+    }
+    if (version < 2) {
+      this.clearThumbnailsFolder();
+    }
 
     this.currentVersion = LATEST_STORAGE_VERSION;
   },
 
   removeThumbnailsFromRoamingProfile:
   function Migrator_removeThumbnailsFromRoamingProfile() {
     let local = FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY]);
     let roaming = FileUtils.getDir("ProfD", [THUMBNAIL_DIRECTORY]);
 
     if (!roaming.equals(local) && roaming.exists()) {
+      roaming.followLinks = false;
       try {
         roaming.remove(true);
       } catch (e) {
         // The directory might not exist or we're not permitted to remove it.
       }
     }
+  },
+
+  clearThumbnailsFolder: function Migrator_clearThumbnailsFolder() {
+    let dir = FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY]);
+    dir.followLinks = false;
+    try {
+      dir.remove(true);
+    } catch (e) {
+      // The directory might not exist or we're not permitted to remove it.
+    }
+  }
+};
+
+let PageThumbsExpiration = {
+  _filters: [],
+
+  init: function Expiration_init() {
+    gUpdateTimerManager.registerTimer("browser-cleanup-thumbnails", this,
+                                      EXPIRATION_INTERVAL_SECS);
+  },
+
+  addFilter: function Expiration_addFilter(aFilter) {
+    this._filters.push(aFilter);
+  },
+
+  removeFilter: function Expiration_removeFilter(aFilter) {
+    let index = this._filters.indexOf(aFilter);
+    if (index > -1)
+      this._filters.splice(index, 1);
+  },
+
+  notify: function Expiration_notify(aTimer) {
+    let urls = [];
+    let filtersToWaitFor = this._filters.length;
+
+    let expire = function expire() {
+      this.expireThumbnails(urls);
+    }.bind(this);
+
+    // No registered filters.
+    if (!filtersToWaitFor) {
+      expire();
+      return;
+    }
+
+    function filterCallback(aURLs) {
+      urls = urls.concat(aURLs);
+      if (--filtersToWaitFor == 0)
+        expire();
+    }
+
+    for (let filter of this._filters) {
+      if (typeof filter == "function")
+        filter(filterCallback)
+      else
+        filter.filterForThumbnailExpiration(filterCallback);
+    }
+  },
+
+  expireThumbnails: function Expiration_expireThumbnails(aURLsToKeep) {
+    let keep = {};
+
+    // Transform all these URLs into file names.
+    for (let url of aURLsToKeep) {
+      keep[PageThumbsStorage.getLeafNameForURL(url)] = true;
+    }
+
+    let numFilesRemoved = 0;
+    let dir = PageThumbsStorage.getDirectory().path;
+    let msg = {type: "getFilesInDirectory", path: dir};
+
+    PageThumbsWorker.postMessage(msg, function (aData) {
+      let files = [file for (file of aData.result) if (!(file in keep))];
+      let maxFilesToRemove = Math.max(EXPIRATION_MIN_CHUNK_SIZE,
+                                      Math.round(files.length / 2));
+
+      let fileNames = files.slice(0, maxFilesToRemove);
+      let filePaths = [dir + "/" + fileName for (fileName of fileNames)];
+      PageThumbsWorker.postMessage({type: "removeFiles", paths: filePaths});
+    });
+  }
+};
+
+/**
+ * Interface to a dedicated thread handling I/O
+ */
+let PageThumbsWorker = {
+  /**
+   * A (fifo) queue of callbacks registered for execution
+   * upon completion of calls to the worker.
+   */
+  _callbacks: [],
+
+  /**
+   * Get the worker, spawning it if necessary.
+   * Code of the worker is in companion file PageThumbsWorker.js
+   */
+  get _worker() {
+    delete this._worker;
+    this._worker = new ChromeWorker("resource://gre/modules/PageThumbsWorker.js");
+    this._worker.addEventListener("message", this);
+    return this._worker;
+  },
+
+  /**
+   * Post a message to the dedicated thread, registering a callback
+   * to be executed once the reply has been received.
+   *
+   * See PageThumbsWorker.js for the format of messages and replies.
+   *
+   * @param {*} message A JSON message.
+   * @param {Function=} callback An optional callback.
+   */
+  postMessage: function Worker_postMessage(message, callback) {
+    this._callbacks.push(callback);
+    this._worker.postMessage(message);
+  },
+
+  /**
+   * Handle a message from the dedicated thread.
+   */
+  handleEvent: function Worker_handleEvent(aEvent) {
+    let callback = this._callbacks.shift();
+    if (callback)
+      callback(aEvent.data);
   }
 };
 
 let PageThumbsHistoryObserver = {
   onDeleteURI: function Thumbnails_onDeleteURI(aURI, aGUID) {
     PageThumbsStorage.remove(aURI.spec);
   },
 
new file mode 100644
--- /dev/null
+++ b/browser/components/thumbnails/PageThumbsWorker.js
@@ -0,0 +1,64 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * A worker dedicated for the I/O component of PageThumbs storage.
+ *
+ * Do not rely on the API of this worker. In a future version, it might be
+ * fully replaced by a OS.File global I/O worker.
+ */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm");
+
+let PageThumbsWorker = {
+  handleMessage: function Worker_handleMessage(aEvent) {
+    let msg = aEvent.data;
+    let data = {result: null, data: null};
+
+    switch (msg.type) {
+      case "removeFiles":
+        data.result = this.removeFiles(msg);
+        break;
+      case "getFilesInDirectory":
+        data.result = this.getFilesInDirectory(msg);
+        break;
+      default:
+        data.result = false;
+        data.detail = "message not understood";
+        break;
+    }
+
+    self.postMessage(data);
+  },
+
+  getFilesInDirectory: function Worker_getFilesInDirectory(msg) {
+    let iter = new OS.File.DirectoryIterator(msg.path);
+    let entries = [];
+
+    for (let entry in iter) {
+      if (!entry.isDir && !entry.isSymLink) {
+        entries.push(entry.name);
+      }
+    }
+
+    iter.close();
+    return entries;
+  },
+
+  removeFiles: function Worker_removeFiles(msg) {
+    for (let file of msg.paths) {
+      try {
+        OS.File.remove(file);
+      } catch (e) {
+        // We couldn't remove the file for some reason.
+        // Let's just continue with the next one.
+      }
+    }
+    return true;
+  }
+};
+
+self.onmessage = PageThumbsWorker.handleMessage.bind(PageThumbsWorker);
--- a/browser/components/thumbnails/test/Makefile.in
+++ b/browser/components/thumbnails/test/Makefile.in
@@ -12,17 +12,16 @@ include $(DEPTH)/config/autoconf.mk
 include $(topsrcdir)/config/rules.mk
 
 _BROWSER_FILES = \
 	browser_thumbnails_capture.js \
 	browser_thumbnails_privacy.js \
 	browser_thumbnails_redirect.js \
 	browser_thumbnails_storage.js \
 	browser_thumbnails_bug726727.js \
-	browser_thumbnails_bug753755.js \
 	head.js \
 	background_red.html \
 	background_red_redirect.sjs \
 	privacy_cache_control.sjs \
 	$(NULL)
 
 libs::	$(_BROWSER_FILES)
 	$(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/browser/$(relativesrcdir)
deleted file mode 100644
--- a/browser/components/thumbnails/test/browser_thumbnails_bug753755.js
+++ /dev/null
@@ -1,15 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
-   http://creativecommons.org/publicdomain/zero/1.0/ */
-
-/**
- * Make sure that PageThumbsStorage.getFileForURL(url) doesn't implicitly
- * create the file's parent path.
- */
-function runTests() {
-  let url = "http://non.existant.url/";
-  let file = PageThumbsStorage.getFileForURL(url);
-  ok(!file.exists() && !file.parent.exists(), "file and path don't exist");
-
-  let file = PageThumbsStorage.getFileForURL(url, {createPath: true});
-  ok(!file.exists() && file.parent.exists(), "path exists, file doesn't");
-}
--- a/browser/modules/NewTabUtils.jsm
+++ b/browser/modules/NewTabUtils.jsm
@@ -11,16 +11,19 @@ const Cc = Components.classes;
 const Cu = Components.utils;
 
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
   "resource://gre/modules/PlacesUtils.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
+  "resource:///modules/PageThumbs.jsm");
+
 XPCOMUtils.defineLazyGetter(this, "gPrincipal", function () {
   let uri = Services.io.newURI("about:newtab", null, null);
   return Services.scriptSecurityManager.getNoAppCodebasePrincipal(uri);
 });
 
 // The preference that tells whether this feature is enabled.
 const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled";
 
@@ -606,18 +609,16 @@ let Telemetry = {
   /**
    * Listens for gather telemetry topic.
    */
   observe: function Telemetry_observe(aSubject, aTopic, aData) {
     this._collect();
   }
 };
 
-Telemetry.init();
-
 /**
  * Singleton that checks if a given link should be displayed on about:newtab
  * or if we should rather not do it for security reasons. URIs that inherit
  * their caller's principal will be filtered.
  */
 let LinkChecker = {
   _cache: {},
 
@@ -639,20 +640,56 @@ let LinkChecker = {
       return true;
     } catch (e) {
       // We got a weird URI or one that would inherit the caller's principal.
       return false;
     }
   }
 };
 
+let ExpirationFilter = {
+  init: function ExpirationFilter_init() {
+    PageThumbs.addExpirationFilter(this);
+  },
+
+  filterForThumbnailExpiration:
+  function ExpirationFilter_filterForThumbnailExpiration(aCallback) {
+    if (!AllPages.enabled) {
+      aCallback([]);
+      return;
+    }
+
+    Links.populateCache(function () {
+      let urls = [];
+
+      // Add all URLs to the list that we want to keep thumbnails for.
+      for (let link of Links.getLinks().slice(0, 25)) {
+        if (link && link.url)
+          urls.push(link.url);
+      }
+
+      aCallback(urls);
+    });
+  }
+};
+
 /**
  * Singleton that provides the public API of this JSM.
  */
 let NewTabUtils = {
+  _initialized: false,
+
+  init: function NewTabUtils_init() {
+    if (!this._initialized) {
+      this._initialized = true;
+      ExpirationFilter.init();
+      Telemetry.init();
+    }
+  },
+
   /**
    * Restores all sites that have been removed from the grid.
    */
   restore: function NewTabUtils_restore() {
     Storage.clear();
     Links.resetCache();
     PinnedLinks.resetCache();
     BlockedLinks.resetCache();