Bug 1159571 - Use server provided frequency caps for daily and lifetime totals [r=adw]
authorMaxim Zhilyaev <mzhilyaev@mozilla.com>
Fri, 08 May 2015 09:20:18 -0700
changeset 274511 ded137b6928344c36f2f6c69038546bedc3fbe7d
parent 274510 d9a7a06ee0b5e4053dced334e43b75cbc5568081
child 274512 e471fd92e2bbe9ba1e8338776398daeaf87c4b7c
push id863
push userraliiev@mozilla.com
push dateMon, 03 Aug 2015 13:22:43 +0000
treeherdermozilla-release@f6321b14228d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersadw
bugs1159571
milestone40.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1159571 - Use server provided frequency caps for daily and lifetime totals [r=adw]
browser/docs/DirectoryLinksProvider.rst
browser/modules/DirectoryLinksProvider.jsm
browser/modules/test/xpcshell/test_DirectoryLinksProvider.js
toolkit/modules/NewTabUtils.jsm
--- a/browser/docs/DirectoryLinksProvider.rst
+++ b/browser/docs/DirectoryLinksProvider.rst
@@ -143,16 +143,17 @@ Below is an example directory source fil
                   "hg.mozilla.org",
                   "mozilla.org",
                   "planet.mozilla.org",
                   "quality.mozilla.org",
                   "support.mozilla.org",
                   "treeherder.mozilla.org",
                   "wiki.mozilla.org"
               ],
+              "frequency_caps": {"daily": 3, "total": 10},
               "imageURI": "https://tiles.cdn.mozilla.net/images/9ee2b265678f2775de2e4bf680df600b502e6038.3875.png",
               "time_limits": {"start": "2014-01-01T00:00:00.000Z", "end": "2014-02-01T00:00:00.000Z"},
               "title": "Thanks for testing!",
               "type": "affiliate",
               "url": "https://www.mozilla.com/firefox/tiles"
           }
       ]
   }
@@ -178,16 +179,19 @@ Suggested Link Object Extras
 ----------------------------
 
 A suggested link has additional values:
 
 - ``frecent_sites`` - array of strings of the sites that can trigger showing a
   Suggested Tile if the user has the site in one of the top 100 most-frecent
   pages. Only preapproved array of strings that are hardcoded into the
   DirectoryLinksProvider module are allowed.
+- ``frequency_caps`` - an object consisting of daily and total frequency caps
+  that limit the number of times a Suggested Tile can be shown in the new tab
+  per day and overall.
 - ``time_limits`` - an object consisting of start and end timestamps specifying
   when a Suggested Tile may start and has to stop showing in the newtab.
   The timestamp is expected in ISO_8601 format: '2014-01-10T20:00:00.000Z'
 
 The preapproved arrays follow a policy for determining what topic grouping is
 allowed as well as the composition of a grouping. The topics are broad
 uncontroversial categories, e.g., Mobile Phone, News, Technology, Video Game,
 Web Development. There are at least 5 sites within a grouping, and as many
--- a/browser/modules/DirectoryLinksProvider.jsm
+++ b/browser/modules/DirectoryLinksProvider.jsm
@@ -85,18 +85,26 @@ const ALLOWED_LINK_SCHEMES = new Set(["h
 const ALLOWED_IMAGE_SCHEMES = new Set(["https", "data"]);
 
 // The frecency of a directory link
 const DIRECTORY_FRECENCY = 1000;
 
 // The frecency of a suggested link
 const SUGGESTED_FRECENCY = Infinity;
 
-// Default number of times to show a link
-const DEFAULT_FREQUENCY_CAP = 5;
+// The filename where frequency cap data stored locally
+const FREQUENCY_CAP_FILE = "frequencyCap.json";
+
+// Default settings for daily and total frequency caps
+const DEFAULT_DAILY_FREQUENCY_CAP = 3;
+const DEFAULT_TOTAL_FREQUENCY_CAP = 10;
+
+// Default timeDelta to prune unused frequency cap objects
+// currently set to 10 days in milliseconds
+const DEFAULT_PRUNE_TIME_DELTA = 10*24*60*60*1000;
 
 // The min number of visible (not blocked) history tiles to have before showing suggested tiles
 const MIN_VISIBLE_HISTORY_TILES = 8;
 
 // Divide frecency by this amount for pings
 const PING_SCORE_DIVISOR = 10000;
 
 // Allowed ping actions remotely stored as columns: case-insensitive [a-z0-9_]
@@ -120,26 +128,26 @@ let DirectoryLinksProvider = {
   _downloadIntervalMS: 86400000,
 
   /**
    * A mapping from eTLD+1 to an enhanced link objects
    */
   _enhancedLinks: new Map(),
 
   /**
-   * A mapping from site to remaining number of views
-   */
-  _frequencyCaps: new Map(),
-
-  /**
    * A mapping from site to a list of suggested link objects
    */
   _suggestedLinks: new Map(),
 
   /**
+   * Frequency Cap object - maintains daily and total tile counts, and frequency cap settings
+   */
+  _frequencyCaps: {},
+
+  /**
    * A set of top sites that we can provide suggested links for
    */
   _topSitesWithSuggestedLinks: new Set(),
 
   get _observedPrefs() Object.freeze({
     enhanced: PREF_NEWTAB_ENHANCED,
     linksURL: PREF_DIRECTORY_SOURCE,
     matchOSLocale: PREF_MATCH_OS_LOCALE,
@@ -465,25 +473,25 @@ let DirectoryLinksProvider = {
    * @return download promise
    */
   reportSitesAction: function DirectoryLinksProvider_reportSitesAction(sites, action, triggeringSiteIndex) {
     // Check if the suggested tile was shown
     if (action == "view") {
       sites.slice(0, triggeringSiteIndex + 1).forEach(site => {
         let {targetedSite, url} = site.link;
         if (targetedSite) {
-          this._decreaseFrequencyCap(url, 1);
+          this._addFrequencyCapView(url);
         }
       });
     }
     // Use up all views if the user clicked on a frequency capped tile
     else if (action == "click") {
       let {targetedSite, url} = sites[triggeringSiteIndex].link;
       if (targetedSite) {
-        this._decreaseFrequencyCap(url, DEFAULT_FREQUENCY_CAP);
+        this._setFrequencyCapClick(url);
       }
     }
 
     let newtabEnhanced = false;
     let pingEndPoint = "";
     try {
       newtabEnhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED);
       pingEndPoint = Services.prefs.getCharPref(PREF_DIRECTORY_PING);
@@ -528,18 +536,22 @@ let DirectoryLinksProvider = {
       data[action] = actionIndex;
     }
 
     // Package the data to be sent with the ping
     let ping = new XMLHttpRequest();
     ping.open("POST", pingEndPoint + (action == "view" ? "view" : "click"));
     ping.send(JSON.stringify(data));
 
-    // Use this as an opportunity to potentially fetch new links
-    return this._fetchAndCacheLinksIfNecessary();
+    return Task.spawn(function* () {
+      // since we updated views/clicks we need write _frequencyCaps to disk
+      yield this._writeFrequencyCapFile();
+      // Use this as an opportunity to potentially fetch new links
+      yield this._fetchAndCacheLinksIfNecessary();
+    }.bind(this));
   },
 
   /**
    * Get the enhanced link object for a link (whether history or directory)
    */
   getEnhancedLink: function DirectoryLinksProvider_getEnhancedLink(link) {
     // Use the provided link if it's already enhanced
     return link.enhancedImageURI && link ? link :
@@ -575,17 +587,16 @@ let DirectoryLinksProvider = {
   /**
    * Gets the current set of directory links.
    * @param aCallback The function that the array of links is passed to.
    */
   getLinks: function DirectoryLinksProvider_getLinks(aCallback) {
     this._readDirectoryLinksFile().then(rawLinks => {
       // Reset the cache of suggested tiles and enhanced images for this new set of links
       this._enhancedLinks.clear();
-      this._frequencyCaps.clear();
       this._suggestedLinks.clear();
       this._clearCampaignTimeout();
 
       let validityFilter = function(link) {
         // Make sure the link url is allowed and images too if they exist
         return this.isURLAllowed(link.url, ALLOWED_LINK_SCHEMES) &&
                this.isURLAllowed(link.imageURI, ALLOWED_IMAGE_SCHEMES) &&
                this.isURLAllowed(link.enhancedImageURI, ALLOWED_IMAGE_SCHEMES);
@@ -599,17 +610,17 @@ let DirectoryLinksProvider = {
         }
 
         link.targetedName = name;
         link.lastVisitDate = rawLinks.suggested.length - position;
 
         // We cache suggested tiles here but do not push any of them in the links list yet.
         // The decision for which suggested tile to include will be made separately.
         this._cacheSuggestedLinks(link);
-        this._frequencyCaps.set(link.url, DEFAULT_FREQUENCY_CAP);
+        this._updateFrequencyCapSettings(link);
       });
 
       rawLinks.enhanced.filter(validityFilter).forEach((link, position) => {
         link.lastVisitDate = rawLinks.enhanced.length - position;
 
         // Stash the enhanced image for the site
         if (link.enhancedImageURI) {
           this._enhancedLinks.set(NewTabUtils.extractSite(link.url), link);
@@ -620,16 +631,21 @@ let DirectoryLinksProvider = {
         link.lastVisitDate = rawLinks.directory.length - position;
         link.frecency = DIRECTORY_FRECENCY;
         return link;
       });
 
       // Allow for one link suggestion on top of the default directory links
       this.maxNumLinks = links.length + 1;
 
+      // prune frequency caps of outdated urls
+      this._pruneFrequencyCapUrls();
+      // write frequency caps object to disk asynchronously
+      this._writeFrequencyCapFile();
+
       return links;
     }).catch(ex => {
       Cu.reportError(ex);
       return [];
     }).then(links => {
       aCallback(links);
       this._populatePlacesLinks();
     });
@@ -637,26 +653,31 @@ let DirectoryLinksProvider = {
 
   init: function DirectoryLinksProvider_init() {
     this._setDefaultEnhanced();
     this._addPrefsObserver();
     // setup directory file path and last download timestamp
     this._directoryFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, DIRECTORY_LINKS_FILE);
     this._lastDownloadMS = 0;
 
+    // setup frequency cap file path
+    this._frequencyCapFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, FREQUENCY_CAP_FILE);
+
     NewTabUtils.placesProvider.addObserver(this);
     NewTabUtils.links.addObserver(this);
 
     return Task.spawn(function() {
       // get the last modified time of the links file if it exists
       let doesFileExists = yield OS.File.exists(this._directoryFilePath);
       if (doesFileExists) {
         let fileInfo = yield OS.File.stat(this._directoryFilePath);
         this._lastDownloadMS = Date.parse(fileInfo.lastModificationDate);
       }
+      // read frequency cap file
+      yield this._readFrequencyCapFile();
       // fetch directory on startup without force
       yield this._fetchAndCacheLinksIfNecessary();
     }.bind(this));
   },
 
   _handleManyLinksChanged: function() {
     this._topSitesWithSuggestedLinks.clear();
     this._suggestedLinks.forEach((suggestedLinks, site) => {
@@ -690,47 +711,48 @@ let DirectoryLinksProvider = {
   },
 
   _populatePlacesLinks: function () {
     NewTabUtils.links.populateProviderCache(NewTabUtils.placesProvider, () => {
       this._handleManyLinksChanged();
     });
   },
 
+  onDeleteURI: function(aProvider, aLink) {
+    let {url} = aLink;
+    // remove clicked flag for that url and
+    // call observer upon disk write completion
+    this._removeTileClick(url).then(() => {
+      this._callObservers("onDeleteURI", url);
+    });
+  },
+
+  onClearHistory: function() {
+    // remove all clicked flags and call observers upon file write
+    this._removeAllTileClicks().then(() => {
+      this._callObservers("onClearHistory");
+    });
+  },
+
   onLinkChanged: function (aProvider, aLink) {
     // Make sure NewTabUtils.links handles the notification first.
     setTimeout(() => {
       if (this._handleLinkChanged(aLink) || this._shouldUpdateSuggestedTile()) {
         this._updateSuggestedTile();
       }
     }, 0);
   },
 
   onManyLinksChanged: function () {
     // Make sure NewTabUtils.links handles the notification first.
     setTimeout(() => {
       this._handleManyLinksChanged();
     }, 0);
   },
 
-  /**
-   * Record for a url that some number of views have been used
-   * @param url String url of the suggested link
-   * @param amount Number of equivalent views to decrease
-   */
-  _decreaseFrequencyCap(url, amount) {
-    let remainingViews = this._frequencyCaps.get(url) - amount;
-    this._frequencyCaps.set(url, remainingViews);
-
-    // Reached the number of views, so pick a new one.
-    if (remainingViews <= 0) {
-      this._updateSuggestedTile();
-    }
-  },
-
   _getCurrentTopSiteCount: function() {
     let visibleTopSiteCount = 0;
     for (let link of NewTabUtils.links.getLinks().slice(0, MIN_VISIBLE_HISTORY_TILES)) {
       if (link && (link.type == "history" || link.type == "enhanced")) {
         visibleTopSiteCount++;
       }
     }
     return visibleTopSiteCount;
@@ -800,17 +822,17 @@ let DirectoryLinksProvider = {
     // random from flattenedLinks if it appears only once.
     let nextTimeout;
     let possibleLinks = new Map();
     let targetedSites = new Map();
     this._topSitesWithSuggestedLinks.forEach(topSiteWithSuggestedLink => {
       let suggestedLinksMap = this._suggestedLinks.get(topSiteWithSuggestedLink);
       suggestedLinksMap.forEach((suggestedLink, url) => {
         // Skip this link if we've shown it too many times already
-        if (this._frequencyCaps.get(url) <= 0) {
+        if (!this._testFrequencyCapLimits(url)) {
           return;
         }
 
         // as we iterate suggestedLinks, check for campaign start/end
         // time limits, and set nextTimeout to the closest timestamp
         let {use, timeoutDate} = this._testLinkForCampaignTimeLimits(suggestedLink);
         // update nextTimeout is necessary
         if (timeoutDate && (!nextTimeout || nextTimeout > timeoutDate)) {
@@ -858,16 +880,219 @@ let DirectoryLinksProvider = {
       // store frecency by URL not by site.
       targetedSite: targetedSites.get(chosenSuggestedLink.url).length ?
         targetedSites.get(chosenSuggestedLink.url)[0] : null
     }, chosenSuggestedLink));
     return chosenSuggestedLink;
    },
 
   /**
+   * Reads json file, parses its content, and returns resulting object
+   * @param json file path
+   * @param json object to return in case file read or parse fails
+   * @return a promise resolved to a valid object or undefined upon error
+   */
+  _readJsonFile: Task.async(function* (filePath, nullObject) {
+    let jsonObj;
+    try {
+      let binaryData = yield OS.File.read(filePath);
+      let json = gTextDecoder.decode(binaryData);
+      jsonObj = JSON.parse(json);
+    }
+    catch (e) {}
+    return jsonObj || nullObject;
+  }),
+
+  /**
+   * Loads frequency cap object from file and parses its content
+   * @return a promise resolved upon load completion
+   *         on error or non-exstent file _frequencyCaps is set to empty object
+   */
+  _readFrequencyCapFile: Task.async(function* () {
+    // set _frequencyCaps object to file's content or empty object
+    this._frequencyCaps = yield this._readJsonFile(this._frequencyCapFilePath, {});
+  }),
+
+  /**
+   * Saves frequency cap object to file
+   * @return a promise resolved upon file i/o completion
+   */
+  _writeFrequencyCapFile: function DirectoryLinksProvider_writeFrequencyCapFile() {
+    let json = JSON.stringify(this._frequencyCaps || {});
+    return OS.File.writeAtomic(this._frequencyCapFilePath, json, {tmpPath: this._frequencyCapFilePath + ".tmp"});
+  },
+
+  /**
+   * Clears frequency cap object and writes empty json to file
+   * @return a promise resolved upon file i/o completion
+   */
+  _clearFrequencyCap: function DirectoryLinksProvider_clearFrequencyCap() {
+    this._frequencyCaps = {};
+    return this._writeFrequencyCapFile();
+  },
+
+  /**
+   * updates frequency cap configuration for a link
+   */
+  _updateFrequencyCapSettings: function DirectoryLinksProvider_updateFrequencyCapSettings(link) {
+    let capsObject = this._frequencyCaps[link.url];
+    if (!capsObject) {
+      // create an object with empty counts
+      capsObject = {
+        dailyViews: 0,
+        totalViews: 0,
+        lastShownDate: 0,
+      };
+      this._frequencyCaps[link.url] = capsObject;
+    }
+    // set last updated timestamp
+    capsObject.lastUpdated = Date.now();
+    // check for link configuration
+    if (link.frequency_caps) {
+      capsObject.dailyCap = link.frequency_caps.daily || DEFAULT_DAILY_FREQUENCY_CAP;
+      capsObject.totalCap = link.frequency_caps.total || DEFAULT_TOTAL_FREQUENCY_CAP;
+    }
+    else {
+      // fallback to defaults
+      capsObject.dailyCap = DEFAULT_DAILY_FREQUENCY_CAP;
+      capsObject.totalCap = DEFAULT_TOTAL_FREQUENCY_CAP;
+    }
+  },
+
+  /**
+   * Prunes frequency cap objects for outdated links
+   * @param timeDetla milliseconds
+   *        all cap objects with lastUpdated less than (now() - timeDelta)
+   *        will be removed. This is done to remove frequency cap objects
+   *        for unused tile urls
+   */
+  _pruneFrequencyCapUrls: function DirectoryLinksProvider_pruneFrequencyCapUrls(timeDelta = DEFAULT_PRUNE_TIME_DELTA) {
+    let timeThreshold = Date.now() - timeDelta;
+    Object.keys(this._frequencyCaps).forEach(url => {
+      if (this._frequencyCaps[url].lastUpdated <= timeThreshold) {
+        delete this._frequencyCaps[url];
+      }
+    });
+  },
+
+  /**
+   * Checks if supplied timestamp happened today
+   * @param timestamp in milliseconds
+   * @return true if the timestamp was made today, false otherwise
+   */
+  _wasToday: function DirectoryLinksProvider_wasToday(timestamp) {
+    let showOn = new Date(timestamp);
+    let today = new Date();
+    // call timestamps identical if both day and month are same
+    return showOn.getDate() == today.getDate() &&
+           showOn.getMonth() == today.getMonth() &&
+           showOn.getYear() == today.getYear();
+  },
+
+  /**
+   * adds some number of views for a url
+   * @param url String url of the suggested link
+   */
+  _addFrequencyCapView: function DirectoryLinksProvider_addFrequencyCapView(url) {
+    let capObject = this._frequencyCaps[url];
+    // sanity check
+    if (!capObject) {
+      return;
+    }
+
+    // if the day is new: reset the daily counter and lastShownDate
+    if (!this._wasToday(capObject.lastShownDate)) {
+      capObject.dailyViews = 0;
+      // update lastShownDate
+      capObject.lastShownDate = Date.now();
+    }
+
+    // bump both dialy and total counters
+    capObject.totalViews++;
+    capObject.dailyViews++;
+
+    // if any of the caps is reached - update suggested tiles
+    if (capObject.totalViews >= capObject.totalCap ||
+        capObject.dailyViews >= capObject.dailyCap) {
+      this._updateSuggestedTile();
+    }
+  },
+
+  /**
+   * Sets clicked flag for link url
+   * @param url String url of the suggested link
+   */
+  _setFrequencyCapClick: function DirectoryLinksProvider_reportFrequencyCapClick(url) {
+    let capObject = this._frequencyCaps[url];
+    // sanity check
+    if (!capObject) {
+      return;
+    }
+    capObject.clicked = true;
+    // and update suggested tiles, since current tile became invalid
+    this._updateSuggestedTile();
+  },
+
+  /**
+   * Tests frequency cap limits for link url
+   * @param url String url of the suggested link
+   * @return true if link is viewable, false otherwise
+   */
+  _testFrequencyCapLimits: function DirectoryLinksProvider_testFrequencyCapLimits(url) {
+    let capObject = this._frequencyCaps[url];
+    // sanity check: if url is missing - do not show this tile
+    if (!capObject) {
+      return false;
+    }
+
+    // check for clicked set or total views reached
+    if (capObject.clicked || capObject.totalViews >= capObject.totalCap) {
+      return false;
+    }
+
+    // otherwise check if link is over daily views limit
+    if (this._wasToday(capObject.lastShownDate) &&
+        capObject.dailyViews >= capObject.dailyCap) {
+      return false;
+    }
+
+    // we passed all cap tests: return true
+    return true;
+  },
+
+  /**
+   * Removes clicked flag from frequency cap entry for tile landing url
+   * @param url String url of the suggested link
+   * @return promise resolved upon disk write completion
+   */
+  _removeTileClick: function DirectoryLinksProvider_removeTileClick(url = "") {
+    // remove trailing slash, to accomodate Places sending site urls ending with '/'
+    let noTrailingSlashUrl = url.replace(/\/$/,"");
+    let capObject = this._frequencyCaps[url] || this._frequencyCaps[noTrailingSlashUrl];
+    // return resolved promise if capObject is not found
+    if (!capObject) {
+      return Promise.resolve();;
+    }
+    // otherwise remove clicked flag
+    delete capObject.clicked;
+    return this._writeFrequencyCapFile();
+  },
+
+  /**
+   * Removes all clicked flags from frequency cap object
+   * @return promise resolved upon disk write completion
+   */
+  _removeAllTileClicks: function DirectoryLinksProvider_removeAllTileClicks() {
+    Object.keys(this._frequencyCaps).forEach(url => {
+      delete this._frequencyCaps[url].clicked;
+    });
+    return this._writeFrequencyCapFile();
+  },
+
+  /**
    * Return the object to its pre-init state
    */
   reset: function DirectoryLinksProvider_reset() {
     delete this.__linksURL;
     this._removePrefsObserver();
     this._removeObservers();
   },
 
--- a/browser/modules/test/xpcshell/test_DirectoryLinksProvider.js
+++ b/browser/modules/test/xpcshell/test_DirectoryLinksProvider.js
@@ -11,16 +11,17 @@ const { classes: Cc, interfaces: Ci, res
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource:///modules/DirectoryLinksProvider.jsm");
 Cu.import("resource://gre/modules/Promise.jsm");
 Cu.import("resource://gre/modules/Http.jsm");
 Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://gre/modules/osfile.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
   "resource://gre/modules/NetUtil.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils",
   "resource://gre/modules/NewTabUtils.jsm");
 
 do_get_profile();
 
@@ -192,16 +193,17 @@ function promiseSetupDirectoryLinksProvi
     DirectoryLinksProvider._lastDownloadMS = options.lastDownloadMS || 0;
   });
 }
 
 function promiseCleanDirectoryLinksProvider() {
   return Task.spawn(function() {
     yield promiseDirectoryDownloadOnPrefChange(kLocalePref, "en-US");
     yield promiseDirectoryDownloadOnPrefChange(kSourceUrlPref, kTestURL);
+    yield DirectoryLinksProvider._clearFrequencyCap();
     DirectoryLinksProvider._lastDownloadMS  = 0;
     DirectoryLinksProvider.reset();
   });
 }
 
 function run_test() {
   // Set up a mock HTTP server to serve a directory page
   server = new HttpServer();
@@ -218,16 +220,30 @@ function run_test() {
     DirectoryLinksProvider.reset();
     Services.prefs.clearUserPref(kLocalePref);
     Services.prefs.clearUserPref(kSourceUrlPref);
     Services.prefs.clearUserPref(kPingUrlPref);
     Services.prefs.clearUserPref(kNewtabEnhancedPref);
   });
 }
 
+
+function setTimeout(fun, timeout) {
+  let timer = Components.classes["@mozilla.org/timer;1"]
+                        .createInstance(Components.interfaces.nsITimer);
+  var event = {
+    notify: function (timer) {
+      fun();
+    }
+  };
+  timer.initWithCallback(event, timeout,
+                         Components.interfaces.nsITimer.TYPE_ONE_SHOT);
+  return timer;
+}
+
 add_task(function test_shouldUpdateSuggestedTile() {
   let suggestedLink = {
     targetedSite: "somesite.com"
   };
 
   // DirectoryLinksProvider has no suggested tile and no top sites => no need to update
   do_check_eq(DirectoryLinksProvider._getCurrentTopSiteCount(), 0);
   isIdentical(NewTabUtils.getProviderLinks(), []);
@@ -546,17 +562,18 @@ add_task(function test_frequencyCappedSi
   DirectoryLinksProvider._getCurrentTopSiteCount = () => 8;
 
   let testUrl = "http://frequency.capped/link";
   let targets = ["top.site.com"];
   let data = {
     suggested: [{
       type: "affiliate",
       frecent_sites: targets,
-      url: testUrl
+      url: testUrl,
+      frequency_caps: {daily: 5}
     }],
     directory: [{
       type: "organic",
       url: "http://directory.site/"
     }]
   };
   let dataURI = "data:application/json," + JSON.stringify(data);
 
@@ -1394,8 +1411,264 @@ add_task(function test_setupStartEndTime
   link.time_limits.start = "2015-99999-01T00:00:00"
   DirectoryLinksProvider._setupStartEndTime(link);
   do_check_false(link.startTime);
 
   link.time_limits.start = "20150501T00:00:00"
   DirectoryLinksProvider._setupStartEndTime(link);
   do_check_false(link.startTime);
 });
+
+add_task(function test_DirectoryLinksProvider_frequencyCapSetup() {
+  yield promiseSetupDirectoryLinksProvider();
+  yield DirectoryLinksProvider.init();
+
+  yield promiseCleanDirectoryLinksProvider();
+  yield DirectoryLinksProvider._readFrequencyCapFile();
+  isIdentical(DirectoryLinksProvider._frequencyCaps, {});
+
+  // setup few links
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "1",
+  });
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "2",
+      frequency_caps: {daily: 1, total: 2}
+  });
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "3",
+      frequency_caps: {total: 2}
+  });
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "4",
+      frequency_caps: {daily: 1}
+  });
+  let freqCapsObject = DirectoryLinksProvider._frequencyCaps;
+  let capObject = freqCapsObject["1"];
+  let defaultDaily = capObject.dailyCap;
+  let defaultTotal = capObject.totalCap;
+  // check if we have defaults set
+  do_check_true(capObject.dailyCap > 0);
+  do_check_true(capObject.totalCap > 0);
+  // check if defaults are properly handled
+  do_check_eq(freqCapsObject["2"].dailyCap, 1);
+  do_check_eq(freqCapsObject["2"].totalCap, 2);
+  do_check_eq(freqCapsObject["3"].dailyCap, defaultDaily);
+  do_check_eq(freqCapsObject["3"].totalCap, 2);
+  do_check_eq(freqCapsObject["4"].dailyCap, 1);
+  do_check_eq(freqCapsObject["4"].totalCap, defaultTotal);
+
+  // write object to file
+  yield DirectoryLinksProvider._writeFrequencyCapFile();
+  // empty out freqCapsObject and read file back
+  DirectoryLinksProvider._frequencyCaps = {};
+  yield DirectoryLinksProvider._readFrequencyCapFile();
+  // re-ran tests - they should all pass
+  do_check_eq(freqCapsObject["2"].dailyCap, 1);
+  do_check_eq(freqCapsObject["2"].totalCap, 2);
+  do_check_eq(freqCapsObject["3"].dailyCap, defaultDaily);
+  do_check_eq(freqCapsObject["3"].totalCap, 2);
+  do_check_eq(freqCapsObject["4"].dailyCap, 1);
+  do_check_eq(freqCapsObject["4"].totalCap, defaultTotal);
+
+  // wait a second and prune frequency caps
+  yield new Promise(resolve => {
+    setTimeout(resolve, 1100);
+  });
+
+  // update one link and create another
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "3",
+      frequency_caps: {daily: 1, total: 2}
+  });
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+      url: "7",
+      frequency_caps: {daily: 1, total: 2}
+  });
+  // now prune the ones that have been in the object longer than 1 second
+  DirectoryLinksProvider._pruneFrequencyCapUrls(1000);
+  // make sure all keys but "3" and "7" are deleted
+  Object.keys(DirectoryLinksProvider._frequencyCaps).forEach(key => {
+    do_check_true(key == "3" || key == "7");
+  });
+
+  yield promiseCleanDirectoryLinksProvider();
+});
+
+add_task(function test_DirectoryLinksProvider_getFrequencyCapLogic() {
+  yield promiseSetupDirectoryLinksProvider();
+  yield DirectoryLinksProvider.init();
+
+  // setup suggested links
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: "1",
+    frequency_caps: {daily: 2, total: 4}
+  });
+
+  do_check_true(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+  // exhaust daily views
+  DirectoryLinksProvider._addFrequencyCapView("1")
+  do_check_true(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+  DirectoryLinksProvider._addFrequencyCapView("1")
+  do_check_false(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+
+  // now step into the furture
+  let _wasTodayOrig = DirectoryLinksProvider._wasToday;
+  DirectoryLinksProvider._wasToday = function () {return false;}
+  // exhaust total views
+  DirectoryLinksProvider._addFrequencyCapView("1")
+  do_check_true(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+  DirectoryLinksProvider._addFrequencyCapView("1")
+  // reached totalViews 4, should return false
+  do_check_false(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+
+  // add more views by updating configuration
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: "1",
+    frequency_caps: {daily: 5, total: 10}
+  });
+  // should be true, since we have more total views
+  do_check_true(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+
+  // set click flag
+  DirectoryLinksProvider._setFrequencyCapClick("1");
+  // always false after click
+  do_check_false(DirectoryLinksProvider._testFrequencyCapLimits("1"));
+
+  // use unknown urls and ensure nothing breaks
+  DirectoryLinksProvider._addFrequencyCapView("nosuch.url");
+  DirectoryLinksProvider._setFrequencyCapClick("nosuch.url");
+  // testing unknown url should always return false
+  do_check_false(DirectoryLinksProvider._testFrequencyCapLimits("nosuch.url"));
+
+  // reset _wasToday back to original function
+  DirectoryLinksProvider._wasToday = _wasTodayOrig;
+  yield promiseCleanDirectoryLinksProvider();
+});
+
+add_task(function test_DirectoryLinksProvider_getFrequencyCapReportSiteAction() {
+  yield promiseSetupDirectoryLinksProvider();
+  yield DirectoryLinksProvider.init();
+
+  // setup suggested links
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: "bar.com",
+    frequency_caps: {daily: 2, total: 4}
+  });
+
+  do_check_true(DirectoryLinksProvider._testFrequencyCapLimits("bar.com"));
+  // report site action
+  yield DirectoryLinksProvider.reportSitesAction([{
+    link: {
+      targetedSite: "foo.com",
+      url: "bar.com"
+    },
+    isPinned: function() {return false;},
+  }], "view", 0);
+
+  // read file content and ensure that view counters are updated
+  let data = yield readJsonFile(DirectoryLinksProvider._frequencyCapFilePath);
+  do_check_eq(data["bar.com"].dailyViews, 1);
+  do_check_eq(data["bar.com"].totalViews, 1);
+
+  yield promiseCleanDirectoryLinksProvider();
+});
+
+add_task(function test_DirectoryLinksProvider_ClickRemoval() {
+  yield promiseSetupDirectoryLinksProvider();
+  yield DirectoryLinksProvider.init();
+  let landingUrl = "http://foo.com";
+
+  // setup suggested links
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: landingUrl,
+    frequency_caps: {daily: 2, total: 4}
+  });
+
+  // add views
+  DirectoryLinksProvider._addFrequencyCapView(landingUrl)
+  DirectoryLinksProvider._addFrequencyCapView(landingUrl)
+  // make a click
+  DirectoryLinksProvider._setFrequencyCapClick(landingUrl);
+
+  // views must be 2 and click must be set
+  do_check_eq(DirectoryLinksProvider._frequencyCaps[landingUrl].totalViews, 2);
+  do_check_true(DirectoryLinksProvider._frequencyCaps[landingUrl].clicked);
+
+  // now insert a visit into places
+  yield new Promise(resolve => {
+    PlacesUtils.asyncHistory.updatePlaces(
+      {
+        uri: NetUtil.newURI(landingUrl),
+        title: "HELLO",
+        visits: [{
+          visitDate: Date.now()*1000,
+          transitionType: Ci.nsINavHistoryService.TRANSITION_LINK
+        }]
+      },
+      {
+        handleError: function () {do_check_true(false);},
+        handleResult: function () {},
+        handleCompletion: function () {resolve();}
+      }
+    );
+  });
+
+  function UrlDeletionTester() {
+    this.promise = new Promise(resolve => {
+      this.onDeleteURI = (directoryLinksProvider, link) => {
+        resolve();
+      };
+      this.onClearHistory = (directoryLinksProvider) => {
+        resolve();
+      };
+    });
+  };
+
+  let testObserver = new UrlDeletionTester();
+  DirectoryLinksProvider.addObserver(testObserver);
+
+  PlacesUtils.bhistory.removePage(NetUtil.newURI(landingUrl));
+  yield testObserver.promise;
+  DirectoryLinksProvider.removeObserver(testObserver);
+  // views must be 2 and click should not exist
+  do_check_eq(DirectoryLinksProvider._frequencyCaps[landingUrl].totalViews, 2);
+  do_check_false(DirectoryLinksProvider._frequencyCaps[landingUrl].hasOwnProperty("clicked"));
+
+  // verify that disk written data is kosher
+  let data = yield readJsonFile(DirectoryLinksProvider._frequencyCapFilePath);
+  do_check_eq(data[landingUrl].totalViews, 2);
+  do_check_false(data[landingUrl].hasOwnProperty("clicked"));
+
+  // now test clear history
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: landingUrl,
+    frequency_caps: {daily: 2, total: 4}
+  });
+  DirectoryLinksProvider._updateFrequencyCapSettings({
+    url: "http://bar.com",
+    frequency_caps: {daily: 2, total: 4}
+  });
+
+  DirectoryLinksProvider._setFrequencyCapClick(landingUrl);
+  DirectoryLinksProvider._setFrequencyCapClick("http://bar.com");
+  // both tiles must have clicked
+  do_check_true(DirectoryLinksProvider._frequencyCaps[landingUrl].clicked);
+  do_check_true(DirectoryLinksProvider._frequencyCaps["http://bar.com"].clicked);
+
+  testObserver = new UrlDeletionTester();
+  DirectoryLinksProvider.addObserver(testObserver);
+  // remove all hostory
+  PlacesUtils.bhistory.removeAllPages();
+
+  yield testObserver.promise;
+  DirectoryLinksProvider.removeObserver(testObserver);
+  // no clicks should remain in the cap object
+  do_check_false(DirectoryLinksProvider._frequencyCaps[landingUrl].hasOwnProperty("clicked"));
+  do_check_false(DirectoryLinksProvider._frequencyCaps["http://bar.com"].hasOwnProperty("clicked"));
+
+  // verify that disk written data is kosher
+  data = yield readJsonFile(DirectoryLinksProvider._frequencyCapFilePath);
+  do_check_false(data[landingUrl].hasOwnProperty("clicked"));
+  do_check_false(data["http://bar.com"].hasOwnProperty("clicked"));
+
+  yield promiseCleanDirectoryLinksProvider();
+});
--- a/toolkit/modules/NewTabUtils.jsm
+++ b/toolkit/modules/NewTabUtils.jsm
@@ -672,16 +672,30 @@ let PlacesProvider = {
     this._observers.push(aObserver);
   },
 
   _observers: [],
 
   /**
    * Called by the history service.
    */
+  onDeleteURI: function PlacesProvider_onDeleteURI(aURI, aGUID, aReason) {
+    // let observers remove sensetive data associated with deleted visit
+    this._callObservers("onDeleteURI", {
+      url: aURI.spec,
+    });
+  },
+
+  onClearHistory: function() {
+    this._callObservers("onClearHistory")
+  },
+
+  /**
+   * Called by the history service.
+   */
   onFrecencyChanged: function PlacesProvider_onFrecencyChanged(aURI, aNewFrecency, aGUID, aHidden, aLastVisitDate) {
     // The implementation of the query in getLinks excludes hidden and
     // unvisited pages, so it's important to exclude them here, too.
     if (!aHidden && aLastVisitDate) {
       this._callObservers("onLinkChanged", {
         url: aURI.spec,
         frecency: aNewFrecency,
         lastVisitDate: aLastVisitDate,