Bug 1421302 - Personalization fixes for Pocket. r=Mardak, a=gchang
authorChristian Sadilek <csadilek@mozilla.com>
Tue, 28 Nov 2017 11:17:27 -0500
changeset 442631 2796c4cd23e3206f548bc3a83a9c982377d442ca
parent 442630 1361881f526226b4e8d697f5b81b46d811bd5f56
child 442632 409df5c82beae70a7060bceb2e5072ee78d78c52
push id8280
push userryanvm@gmail.com
push dateThu, 30 Nov 2017 19:39:16 +0000
treeherdermozilla-beta@51795bc51deb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersMardak, gchang
bugs1421302
milestone58.0
Bug 1421302 - Personalization fixes for Pocket. r=Mardak, a=gchang MozReview-Commit-ID: LGFPrrrOUDc
browser/extensions/activity-stream/css/activity-stream-linux.css
browser/extensions/activity-stream/css/activity-stream-mac.css
browser/extensions/activity-stream/css/activity-stream-windows.css
browser/extensions/activity-stream/lib/ActivityStream.jsm
browser/extensions/activity-stream/lib/TopStoriesFeed.jsm
browser/extensions/activity-stream/prerendered/locales/de/activity-stream-strings.js
browser/extensions/activity-stream/test/unit/lib/TopStoriesFeed.test.js
--- a/browser/extensions/activity-stream/css/activity-stream-linux.css
+++ b/browser/extensions/activity-stream/css/activity-stream-linux.css
@@ -1212,45 +1212,44 @@ section.top-sites:not(.collapsed):hover 
         vertical-align: middle;
         width: 12px; }
       .collapsible-section .section-top-bar .info-option-manage button:dir(rtl)::after {
         transform: scaleX(-1); }
 
 .collapsible-section .section-disclaimer {
   color: #4A4A4F;
   font-size: 13px;
-  margin-bottom: 16px; }
+  margin-bottom: 16px;
+  position: relative; }
   .collapsible-section .section-disclaimer .section-disclaimer-text {
     display: inline-block; }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
         width: 224px; } }
     @media (min-width: 544px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 336px; } }
+        width: 340px; } }
     @media (min-width: 800px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 640px; } }
+        width: 610px; } }
   .collapsible-section .section-disclaimer a {
     color: #008EA4;
     padding-left: 3px; }
   .collapsible-section .section-disclaimer button {
     margin-top: 2px;
     offset-inline-end: 0;
-    height: 26px;
+    min-height: 26px;
+    max-width: 130px;
     background: #F9F9FA;
     border: 1px solid #B1B1B3;
     border-radius: 4px;
     cursor: pointer; }
     .collapsible-section .section-disclaimer button:hover:not(.dismiss) {
       box-shadow: 0 0 0 5px #D7D7DB;
       transition: box-shadow 150ms; }
-    @media (min-width: 224px) {
-      .collapsible-section .section-disclaimer button {
-        position: relative; } }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer button {
         position: absolute; } }
 
 .collapsible-section .section-body {
   max-height: 1100px;
   margin: 0 -7px;
   padding: 0 7px; }
@@ -1263,16 +1262,13 @@ section.top-sites:not(.collapsed):hover 
 
 .collapsible-section.animation-enabled .section-body {
   transition: max-height 0.5s cubic-bezier(0.07, 0.95, 0, 1); }
 
 .collapsible-section.collapsed .section-body {
   max-height: 0;
   overflow: hidden; }
 
-.collapsible-section.collapsed .section-disclaimer {
-  position: relative; }
-
 .collapsible-section.collapsed .section-info-option {
   pointer-events: none; }
 
 .collapsible-section:not(.collapsed):hover .info-option-icon {
   opacity: 1; }
--- a/browser/extensions/activity-stream/css/activity-stream-mac.css
+++ b/browser/extensions/activity-stream/css/activity-stream-mac.css
@@ -1212,45 +1212,44 @@ section.top-sites:not(.collapsed):hover 
         vertical-align: middle;
         width: 12px; }
       .collapsible-section .section-top-bar .info-option-manage button:dir(rtl)::after {
         transform: scaleX(-1); }
 
 .collapsible-section .section-disclaimer {
   color: #4A4A4F;
   font-size: 13px;
-  margin-bottom: 16px; }
+  margin-bottom: 16px;
+  position: relative; }
   .collapsible-section .section-disclaimer .section-disclaimer-text {
     display: inline-block; }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
         width: 224px; } }
     @media (min-width: 544px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 336px; } }
+        width: 340px; } }
     @media (min-width: 800px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 640px; } }
+        width: 610px; } }
   .collapsible-section .section-disclaimer a {
     color: #008EA4;
     padding-left: 3px; }
   .collapsible-section .section-disclaimer button {
     margin-top: 2px;
     offset-inline-end: 0;
-    height: 26px;
+    min-height: 26px;
+    max-width: 130px;
     background: #F9F9FA;
     border: 1px solid #B1B1B3;
     border-radius: 4px;
     cursor: pointer; }
     .collapsible-section .section-disclaimer button:hover:not(.dismiss) {
       box-shadow: 0 0 0 5px #D7D7DB;
       transition: box-shadow 150ms; }
-    @media (min-width: 224px) {
-      .collapsible-section .section-disclaimer button {
-        position: relative; } }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer button {
         position: absolute; } }
 
 .collapsible-section .section-body {
   max-height: 1100px;
   margin: 0 -7px;
   padding: 0 7px; }
@@ -1263,16 +1262,13 @@ section.top-sites:not(.collapsed):hover 
 
 .collapsible-section.animation-enabled .section-body {
   transition: max-height 0.5s cubic-bezier(0.07, 0.95, 0, 1); }
 
 .collapsible-section.collapsed .section-body {
   max-height: 0;
   overflow: hidden; }
 
-.collapsible-section.collapsed .section-disclaimer {
-  position: relative; }
-
 .collapsible-section.collapsed .section-info-option {
   pointer-events: none; }
 
 .collapsible-section:not(.collapsed):hover .info-option-icon {
   opacity: 1; }
--- a/browser/extensions/activity-stream/css/activity-stream-windows.css
+++ b/browser/extensions/activity-stream/css/activity-stream-windows.css
@@ -1212,45 +1212,44 @@ section.top-sites:not(.collapsed):hover 
         vertical-align: middle;
         width: 12px; }
       .collapsible-section .section-top-bar .info-option-manage button:dir(rtl)::after {
         transform: scaleX(-1); }
 
 .collapsible-section .section-disclaimer {
   color: #4A4A4F;
   font-size: 13px;
-  margin-bottom: 16px; }
+  margin-bottom: 16px;
+  position: relative; }
   .collapsible-section .section-disclaimer .section-disclaimer-text {
     display: inline-block; }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
         width: 224px; } }
     @media (min-width: 544px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 336px; } }
+        width: 340px; } }
     @media (min-width: 800px) {
       .collapsible-section .section-disclaimer .section-disclaimer-text {
-        width: 640px; } }
+        width: 610px; } }
   .collapsible-section .section-disclaimer a {
     color: #008EA4;
     padding-left: 3px; }
   .collapsible-section .section-disclaimer button {
     margin-top: 2px;
     offset-inline-end: 0;
-    height: 26px;
+    min-height: 26px;
+    max-width: 130px;
     background: #F9F9FA;
     border: 1px solid #B1B1B3;
     border-radius: 4px;
     cursor: pointer; }
     .collapsible-section .section-disclaimer button:hover:not(.dismiss) {
       box-shadow: 0 0 0 5px #D7D7DB;
       transition: box-shadow 150ms; }
-    @media (min-width: 224px) {
-      .collapsible-section .section-disclaimer button {
-        position: relative; } }
     @media (min-width: 416px) {
       .collapsible-section .section-disclaimer button {
         position: absolute; } }
 
 .collapsible-section .section-body {
   max-height: 1100px;
   margin: 0 -7px;
   padding: 0 7px; }
@@ -1263,19 +1262,16 @@ section.top-sites:not(.collapsed):hover 
 
 .collapsible-section.animation-enabled .section-body {
   transition: max-height 0.5s cubic-bezier(0.07, 0.95, 0, 1); }
 
 .collapsible-section.collapsed .section-body {
   max-height: 0;
   overflow: hidden; }
 
-.collapsible-section.collapsed .section-disclaimer {
-  position: relative; }
-
 .collapsible-section.collapsed .section-info-option {
   pointer-events: none; }
 
 .collapsible-section:not(.collapsed):hover .info-option-icon {
   opacity: 1; }
 
 .search-wrapper input:focus {
   box-shadow: 0 0 0 1px #0A84FF; }
--- a/browser/extensions/activity-stream/lib/ActivityStream.jsm
+++ b/browser/extensions/activity-stream/lib/ActivityStream.jsm
@@ -54,24 +54,24 @@ const PREFS_CONFIG = new Map([
     getValue: args => JSON.stringify({
       api_key_pref: "extensions.pocket.oAuthConsumerKey",
       // Use the opposite value as what default value the feed would have used
       hidden: !PREFS_CONFIG.get("feeds.section.topstories").getValue(args),
       provider_header: "pocket_feedback_header",
       provider_description: "pocket_description",
       provider_icon: "pocket",
       provider_name: "Pocket",
-      read_more_endpoint: "https://getpocket.cdn.mozilla.net/explore/trending?src=fx_new_tab",
+      read_more_endpoint: "https://getpocket.com/explore/trending?src=fx_new_tab",
       stories_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=2&consumer_key=$apiKey&locale_lang=${args.locale}`,
       stories_referrer: "https://getpocket.com/recommendations",
       info_link: "https://www.mozilla.org/privacy/firefox/#pocketstories",
       disclaimer_link: "https://getpocket.cdn.mozilla.net/firefox/new_tab_learn_more",
       topics_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/trending-topics?version=2&consumer_key=$apiKey&locale_lang=${args.locale}`,
       show_spocs: false,
-      personalized: false
+      personalized: true
     })
   }],
   ["showSponsored", {
     title: "Show sponsored cards in spoc experiment (show_spocs in topstories.options has to be set to true as well)",
     value: true
   }],
   ["filterAdult", {
     title: "Remove adult pages from sites, highlights, etc.",
--- a/browser/extensions/activity-stream/lib/TopStoriesFeed.jsm
+++ b/browser/extensions/activity-stream/lib/TopStoriesFeed.jsm
@@ -16,18 +16,21 @@ const {SectionsManager} = Cu.import("res
 const {UserDomainAffinityProvider} = Cu.import("resource://activity-stream/lib/UserDomainAffinityProvider.jsm", {});
 const {PersistentCache} = Cu.import("resource://activity-stream/lib/PersistentCache.jsm", {});
 
 XPCOMUtils.defineLazyModuleGetter(this, "perfService", "resource://activity-stream/common/PerfService.jsm");
 
 const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
 const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours
 const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours
+const MIN_DOMAIN_AFFINITIES_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours
+const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
 const SECTION_ID = "topstories";
 const SPOC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.spoc.impressions";
+const REC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.rec.impressions";
 const MAX_LIFETIME_CAP = 100; // Guard against misconfiguration on the server
 
 this.TopStoriesFeed = class TopStoriesFeed {
   constructor() {
     this.spocsPerNewTabs = 0;
     this.newTabsSinceSpoc = 0;
     this.spocCampaignMap = new Map();
     this.contentUpdateQueue = [];
@@ -45,16 +48,17 @@ this.TopStoriesFeed = class TopStoriesFe
         this.topics_endpoint = this.produceFinalEndpointUrl(options.topics_endpoint, apiKey);
         this.read_more_endpoint = options.read_more_endpoint;
         this.stories_referrer = options.stories_referrer;
         this.personalized = options.personalized;
         this.show_spocs = options.show_spocs;
         this.maxHistoryQueryResults = options.maxHistoryQueryResults;
         this.storiesLastUpdated = 0;
         this.topicsLastUpdated = 0;
+        this.domainAffinitiesLastUpdated = 0;
 
         this.loadCachedData();
         this.fetchStories();
         this.fetchTopics();
 
         Services.obs.addObserver(this, "idle-daily");
       } catch (e) {
         Cu.reportError(`Problem initializing top stories feed: ${e.message}`);
@@ -84,16 +88,17 @@ this.TopStoriesFeed = class TopStoriesFe
       const response = await fetch(this.stories_endpoint);
       if (!response.ok) {
         throw new Error(`Stories endpoint returned unexpected status: ${response.status}`);
       }
 
       const body = await response.json();
       this.updateSettings(body.settings);
       this.stories = this.rotate(this.transform(body.recommendations));
+      this.cleanUpTopRecImpressionPref();
 
       if (this.show_spocs && body.spocs) {
         this.spocCampaignMap = new Map(body.spocs.map(s => [s.id, `${s.campaign_id}`]));
         this.spocs = this.transform(body.spocs).filter(s => s.score >= s.min_score);
         this.cleanUpCampaignImpressionPref();
       }
 
       this.dispatchUpdateEvent(this.storiesLastUpdated, {rows: this.stories});
@@ -110,16 +115,17 @@ this.TopStoriesFeed = class TopStoriesFe
   async loadCachedData() {
     const data = await this.cache.get();
     let stories = data.stories && data.stories.recommendations;
     let topics = data.topics && data.topics.topics;
     let affinities = data.domainAffinities;
     if (this.personalized && affinities && affinities.scores) {
       this.affinityProvider = new UserDomainAffinityProvider(affinities.timeSegments,
         affinities.parameterSets, affinities.maxHistoryQueryResults, affinities.version, affinities.scores);
+      this.domainAffinitiesLastUpdated = affinities._timestamp;
     }
     if (stories && stories.length > 0 && this.storiesLastUpdated === 0) {
       this.updateSettings(data.stories.settings);
       const rows = this.transform(stories);
       this.dispatchUpdateEvent(this.storiesLastUpdated, {rows});
       this.storiesLastUpdated = data.stories._timestamp;
     }
     if (topics && topics.length > 0 && this.topicsLastUpdated === 0) {
@@ -185,80 +191,73 @@ this.TopStoriesFeed = class TopStoriesFe
   updateSettings(settings) {
     if (!this.personalized) {
       return;
     }
 
     this.spocsPerNewTabs = settings.spocsPerNewTabs;
     this.timeSegments = settings.timeSegments;
     this.domainAffinityParameterSets = settings.domainAffinityParameterSets;
+    this.recsExpireTime = settings.recsExpireTime;
     this.version = settings.version;
 
     if (this.affinityProvider && (this.affinityProvider.version !== this.version)) {
       this.resetDomainAffinityScores();
     }
   }
 
   updateDomainAffinityScores() {
-    if (!this.personalized || !this.domainAffinityParameterSets) {
+    if (!this.personalized || !this.domainAffinityParameterSets ||
+      Date.now() - this.domainAffinitiesLastUpdated < MIN_DOMAIN_AFFINITIES_UPDATE_TIME) {
       return;
     }
 
     const start = perfService.absNow();
 
     this.affinityProvider = new UserDomainAffinityProvider(
       this.timeSegments,
       this.domainAffinityParameterSets,
       this.maxHistoryQueryResults,
       this.version);
 
     this.store.dispatch(ac.PerfEvent({
       event: "topstories.domain.affinity.calculation.ms",
       value: Math.round(perfService.absNow() - start)
     }));
 
-    this.cache.set("domainAffinities", this.affinityProvider.getAffinities());
+    const affinities = this.affinityProvider.getAffinities();
+    affinities._timestamp = this.domainAffinitiesLastUpdated = Date.now();
+    this.cache.set("domainAffinities", affinities);
   }
 
   resetDomainAffinityScores() {
     delete this.affinityProvider;
     this.cache.set("domainAffinities", {});
   }
 
-  // If personalization is turned on we have to rotate stories on the client.
-  // An item can only be on top for two iterations (1hr) before it gets moved
-  // to the end. This will later be improved based on interactions/impressions.
+  // If personalization is turned on, we have to rotate stories on the client so that
+  // active stories are at the front of the list, followed by stories that have expired
+  // impressions i.e. have been displayed for longer than recsExpireTime.
   rotate(items) {
     if (!this.personalized || items.length <= 3) {
       return items;
     }
 
-    if (!this.topItems) {
-      this.topItems = new Map();
-    }
-
-    // This avoids an infinite recursion if for some reason the feed stops
-    // changing. Otherwise, there's a chance we'd be rotating forever to
-    // find an item we haven't displayed on top yet.
-    if (this.topItems.size >= items.length) {
-      this.topItems.clear();
-    }
-
-    const guid = items[0].guid;
-    if (!this.topItems.has(guid)) {
-      this.topItems.set(guid, 0);
-    } else {
-      const val = this.topItems.get(guid) + 1;
-      this.topItems.set(guid, val);
-      if (val >= 2) {
-        items.push(items.shift());
-        this.rotate(items);
+    const maxImpressionAge = Math.max(this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, DEFAULT_RECS_EXPIRE_TIME);
+    const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
+    const expired = [];
+    const active = [];
+    for (const item of items) {
+      if (impressions[item.guid] && Date.now() - impressions[item.guid] >= maxImpressionAge) {
+        expired.push(item);
+      } else {
+        active.push(item);
       }
     }
-    return items;
+    return active.concat(expired);
   }
 
   getApiKeyFromPref(apiKeyPref) {
     if (!apiKeyPref) {
       return apiKeyPref;
     }
 
     return this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref);
@@ -296,17 +295,17 @@ this.TopStoriesFeed = class TopStoriesFe
       const updateContent = () => {
         if (!this.spocs || !this.spocs.length) {
           // We have stories but no spocs so there's nothing to do and this update can be
           // removed from the queue.
           return false;
         }
 
         // Filter spocs based on frequency caps
-        const impressions = this.readCampaignImpressionsPref();
+        const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
         const spocs = this.spocs.filter(s => this.isBelowFrequencyCap(impressions, s));
 
         if (!spocs.length) {
           // There's currently no spoc left to display
           return false;
         }
 
         // Create a new array with a spoc inserted at index 2
@@ -363,53 +362,90 @@ this.TopStoriesFeed = class TopStoriesFe
     const campaignCapExceeded = campaignImpressions
       .filter(i => (Date.now() - i) < (campaignCap.period * 1000)).length >= campaignCap.count;
     return !campaignCapExceeded;
   }
 
   // Clean up campaign impression pref by removing all campaigns that are no
   // longer part of the response, and are therefore considered inactive.
   cleanUpCampaignImpressionPref() {
-    const impressions = this.readCampaignImpressionsPref();
     const campaignIds = new Set(this.spocCampaignMap.values());
+    this.cleanUpImpressionPref(id => !campaignIds.has(id), SPOC_IMPRESSION_TRACKING_PREF);
+  }
+
+  // Clean up rec impression pref by removing all stories that are no
+  // longer part of the response.
+  cleanUpTopRecImpressionPref() {
+    const activeStories = new Set(this.stories.map(s => `${s.guid}`));
+    this.cleanUpImpressionPref(id => !activeStories.has(id), REC_IMPRESSION_TRACKING_PREF);
+  }
+
+  /**
+   * Cleans up the provided impression pref (spocs or recs).
+   *
+   * @param isExpired predicate (boolean-valued function) that returns whether or not
+   * the impression for the given key is expired.
+   * @param pref the impression pref to clean up.
+   */
+  cleanUpImpressionPref(isExpired, pref) {
+    const impressions = this.readImpressionsPref(pref);
     let changed = false;
 
     Object
       .keys(impressions)
-      .forEach(cId => {
-        if (!campaignIds.has(cId)) {
+      .forEach(id => {
+        if (isExpired(id)) {
           changed = true;
-          delete impressions[cId];
+          delete impressions[id];
         }
       });
 
     if (changed) {
-      this.writeCampaignImpressionsPref(impressions);
+      this.writeImpressionsPref(pref, impressions);
     }
   }
 
   // Sets a pref mapping campaign IDs to timestamp arrays.
-  // The timestamps represent impressions which we use to calculate frequency caps.
+  // The timestamps represent impressions which are used to calculate frequency caps.
   recordCampaignImpression(campaignId) {
-    let impressions = this.readCampaignImpressionsPref();
+    let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
 
     const timeStamps = impressions[campaignId] || [];
     timeStamps.push(Date.now());
     impressions = Object.assign(impressions, {[campaignId]: timeStamps});
 
-    this.writeCampaignImpressionsPref(impressions);
+    this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions);
   }
 
-  readCampaignImpressionsPref() {
-    const prefVal = this._prefs.get(SPOC_IMPRESSION_TRACKING_PREF);
+  // Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression).
+  // We use these timestamps to guarantee a story doesn't stay on top for longer than
+  // configured in the feed settings (settings.recsExpireTime).
+  recordTopRecImpressions(topItems) {
+    let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
+    let changed = false;
+
+    topItems.forEach(t => {
+      if (!impressions[t]) {
+        changed = true;
+        impressions = Object.assign(impressions, {[t]: Date.now()});
+      }
+    });
+
+    if (changed) {
+      this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions);
+    }
+  }
+
+  readImpressionsPref(pref) {
+    const prefVal = this._prefs.get(pref);
     return prefVal ? JSON.parse(prefVal) : {};
   }
 
-  writeCampaignImpressionsPref(impressions) {
-    this._prefs.set(SPOC_IMPRESSION_TRACKING_PREF, JSON.stringify(impressions));
+  writeImpressionsPref(pref, impressions) {
+    this._prefs.set(pref, JSON.stringify(impressions));
   }
 
   onAction(action) {
     switch (action.type) {
       case at.INIT:
         this.init();
         break;
       case at.SYSTEM_TICK:
@@ -440,26 +476,37 @@ this.TopStoriesFeed = class TopStoriesFe
       case at.PLACES_HISTORY_CLEARED:
         if (this.personalized) {
           this.resetDomainAffinityScores();
         }
         break;
       case at.TELEMETRY_IMPRESSION_STATS: {
         const payload = action.data;
         const viewImpression = !("click" in payload || "block" in payload || "pocket" in payload);
-        if (this.shouldShowSpocs() && payload.tiles && viewImpression) {
-          payload.tiles.forEach(t => {
-            if (this.spocCampaignMap.has(t.id)) {
-              this.recordCampaignImpression(this.spocCampaignMap.get(t.id));
-            }
-          });
+        if (payload.tiles && viewImpression) {
+          if (this.shouldShowSpocs()) {
+            payload.tiles.forEach(t => {
+              if (this.spocCampaignMap.has(t.id)) {
+                this.recordCampaignImpression(this.spocCampaignMap.get(t.id));
+              }
+            });
+          }
+          if (this.personalized) {
+            const topRecs = payload.tiles
+              .filter(t => !this.spocCampaignMap.has(t.id))
+              .map(t => t.id);
+            this.recordTopRecImpressions(topRecs);
+          }
         }
         break;
       }
     }
   }
 };
 
 this.STORIES_UPDATE_TIME = STORIES_UPDATE_TIME;
 this.TOPICS_UPDATE_TIME = TOPICS_UPDATE_TIME;
 this.SECTION_ID = SECTION_ID;
 this.SPOC_IMPRESSION_TRACKING_PREF = SPOC_IMPRESSION_TRACKING_PREF;
-this.EXPORTED_SYMBOLS = ["TopStoriesFeed", "STORIES_UPDATE_TIME", "TOPICS_UPDATE_TIME", "SECTION_ID", "SPOC_IMPRESSION_TRACKING_PREF"];
+this.REC_IMPRESSION_TRACKING_PREF = REC_IMPRESSION_TRACKING_PREF;
+this.MIN_DOMAIN_AFFINITIES_UPDATE_TIME = MIN_DOMAIN_AFFINITIES_UPDATE_TIME;
+this.DEFAULT_RECS_EXPIRE_TIME = DEFAULT_RECS_EXPIRE_TIME;
+this.EXPORTED_SYMBOLS = ["TopStoriesFeed", "STORIES_UPDATE_TIME", "TOPICS_UPDATE_TIME", "SECTION_ID", "SPOC_IMPRESSION_TRACKING_PREF", "MIN_DOMAIN_AFFINITIES_UPDATE_TIME", "REC_IMPRESSION_TRACKING_PREF", "DEFAULT_RECS_EXPIRE_TIME"];
--- a/browser/extensions/activity-stream/prerendered/locales/de/activity-stream-strings.js
+++ b/browser/extensions/activity-stream/prerendered/locales/de/activity-stream-strings.js
@@ -34,17 +34,17 @@ window.gActivityStreamStrings = {
   "search_button": "Suchen",
   "search_header": "{search_engine_name}-Suche",
   "search_web_placeholder": "Das Web durchsuchen",
   "search_settings": "Sucheinstellungen ändern",
   "section_info_option": "Info",
   "section_info_send_feedback": "Feedback senden",
   "section_info_privacy_notice": "Datenschutzhinweis",
   "section_disclaimer_topstories": "Die interessantesten Inhalte aus dem Internet auf Sie abgestimmt. Von Pocket, jetzt Teil von Mozilla.",
-  "section_disclaimer_topstories_linktext": "Lernen Sie wie das funktioniert.",
+  "section_disclaimer_topstories_linktext": "Lernen Sie, wie das funktioniert.",
   "section_disclaimer_topstories_buttontext": "Okay, verstanden",
   "welcome_title": "Willkommen im neuen Tab",
   "welcome_body": "Firefox nutzt diesen Bereich, um Ihnen Ihre wichtigsten Lesezeichen, Artikel, Videos und kürzlich besuchten Seiten anzuzeigen, damit Sie diese einfach wiederfinden.",
   "welcome_label": "Auswahl Ihrer wichtigsten Seiten",
   "time_label_less_than_minute": "< 1 min",
   "time_label_minute": "{number} m",
   "time_label_hour": "{number} h",
   "time_label_day": "{number} t",
--- a/browser/extensions/activity-stream/test/unit/lib/TopStoriesFeed.test.js
+++ b/browser/extensions/activity-stream/test/unit/lib/TopStoriesFeed.test.js
@@ -5,16 +5,19 @@ const {actionTypes: at} = require("commo
 const {GlobalOverrider} = require("test/unit/utils");
 
 describe("Top Stories Feed", () => {
   let TopStoriesFeed;
   let STORIES_UPDATE_TIME;
   let TOPICS_UPDATE_TIME;
   let SECTION_ID;
   let SPOC_IMPRESSION_TRACKING_PREF;
+  let REC_IMPRESSION_TRACKING_PREF;
+  let MIN_DOMAIN_AFFINITIES_UPDATE_TIME;
+  let DEFAULT_RECS_EXPIRE_TIME;
   let instance;
   let clock;
   let globals;
   let sectionsManagerStub;
   let shortURLStub;
 
   const FAKE_OPTIONS = {
     "stories_endpoint": "https://somedomain.org/stories?key=$apiKey",
@@ -57,17 +60,20 @@ describe("Top Stories Feed", () => {
       }
     }
 
     ({
       TopStoriesFeed,
       STORIES_UPDATE_TIME,
       TOPICS_UPDATE_TIME,
       SECTION_ID,
-      SPOC_IMPRESSION_TRACKING_PREF
+      SPOC_IMPRESSION_TRACKING_PREF,
+      REC_IMPRESSION_TRACKING_PREF,
+      MIN_DOMAIN_AFFINITIES_UPDATE_TIME,
+      DEFAULT_RECS_EXPIRE_TIME
     } = injector({
       "lib/ActivityStreamPrefs.jsm": {Prefs: FakePrefs},
       "lib/ShortURL.jsm": {shortURL: shortURLStub},
       "lib/UserDomainAffinityProvider.jsm": {UserDomainAffinityProvider: FakeUserDomainAffinityProvider},
       "lib/SectionsManager.jsm": {SectionsManager: sectionsManagerStub}
     }));
 
     instance = new TopStoriesFeed();
@@ -332,53 +338,97 @@ describe("Top Stories Feed", () => {
       assert.notCalled(instance.compareScore);
     });
     it("should sort items based on relevance score", () => {
       let items = [{"score": 0.1}, {"score": 0.2}];
       items = items.sort(instance.compareScore);
       assert.deepEqual(items, [{"score": 0.2}, {"score": 0.1}]);
     });
     it("should rotate items if personalization is preffed on", () => {
-      let items = [{"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}];
-
+      let items = [{"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}, {"guid": "g5"}, {"guid": "g6"}];
       instance.personalized = true;
 
+      // No impressions should leave items unchanged
       let rotated = instance.rotate(items);
-      assert.deepEqual(new Map([["g1", 0]]), instance.topItems);
       assert.deepEqual(items, rotated);
 
+      // Recent impression should leave items unchanged
+      instance._prefs.get = pref => (pref === REC_IMPRESSION_TRACKING_PREF) && JSON.stringify({"g1": 1, "g2": 1, "g3": 1});
       rotated = instance.rotate(items);
-      assert.deepEqual(new Map([["g1", 1]]), instance.topItems);
       assert.deepEqual(items, rotated);
 
+      // Impression older than expiration time should rotate items
+      clock.tick(DEFAULT_RECS_EXPIRE_TIME + 1);
       rotated = instance.rotate(items);
-      assert.deepEqual(new Map([["g1", 2], ["g2", 0]]), instance.topItems);
-      assert.deepEqual([{"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}, {"guid": "g1"}], rotated);
+      assert.deepEqual([{"guid": "g4"}, {"guid": "g5"}, {"guid": "g6"}, {"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}], rotated);
 
-      // Simulate g1 on top again which should again be rotated to the end
-      rotated = instance.rotate([{"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}]);
-      assert.deepEqual(new Map([["g1", 3], ["g2", 1]]), instance.topItems);
-      assert.deepEqual([{"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}, {"guid": "g1"}], rotated);
+      instance._prefs.get = pref => (pref === REC_IMPRESSION_TRACKING_PREF) &&
+        JSON.stringify({"g1": 1, "g2": 1, "g3": 1, "g4": DEFAULT_RECS_EXPIRE_TIME + 1});
+      clock.tick(DEFAULT_RECS_EXPIRE_TIME);
+      rotated = instance.rotate(items);
+      assert.deepEqual([{"guid": "g5"}, {"guid": "g6"}, {"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}], rotated);
     });
     it("should not rotate items if personalization is preffed off", () => {
       let items = [{"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}];
 
       instance.personalized = false;
 
-      instance.topItems = new Map([["g1", 1]]);
-      const rotated = instance.rotate(items);
+      instance._prefs.get = pref => (pref === REC_IMPRESSION_TRACKING_PREF) && JSON.stringify({"g1": 1, "g2": 1, "g3": 1});
+      clock.tick(DEFAULT_RECS_EXPIRE_TIME + 1);
+      let rotated = instance.rotate(items);
       assert.deepEqual(items, rotated);
     });
-    it("should stop rotating if all items have been on top", () => {
-      let items = [{"guid": "g1"}, {"guid": "g2"}, {"guid": "g3"}, {"guid": "g4"}];
-      instance.topItems = new Map([["g1", 2], ["g2", 2], ["g3", 2], ["g4", 2]]);
+    it("should record top story impressions", async () => {
+      instance._prefs = {get: pref => undefined, set: sinon.spy()};
       instance.personalized = true;
 
-      const rotated = instance.rotate(items);
-      assert.deepEqual(items, rotated);
+      clock.tick(1);
+      let expectedPrefValue = JSON.stringify({1: 1, 2: 1, 3: 1});
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {tiles: [{id: 1}, {id: 2}, {id: 3}]}});
+      assert.calledWith(instance._prefs.set.firstCall, REC_IMPRESSION_TRACKING_PREF, expectedPrefValue);
+
+      // Only need to record first impression, so impression pref shouldn't change
+      instance._prefs.get = pref => expectedPrefValue;
+      clock.tick(1);
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {tiles: [{id: 1}, {id: 2}, {id: 3}]}});
+      assert.calledOnce(instance._prefs.set);
+
+      // New first impressions should be added
+      clock.tick(1);
+      let expectedPrefValueTwo = JSON.stringify({1: 1, 2: 1, 3: 1, 4: 3, 5: 3, 6: 3});
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {tiles: [{id: 4}, {id: 5}, {id: 6}]}});
+      assert.calledWith(instance._prefs.set.secondCall, REC_IMPRESSION_TRACKING_PREF, expectedPrefValueTwo);
+    });
+    it("should not record top story impressions for non-view impressions", async () => {
+      instance._prefs = {get: pref => undefined, set: sinon.spy()};
+      instance.personalized = true;
+
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {click: 0, tiles: [{id: 1}]}});
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {block: 0, tiles: [{id: 1}]}});
+      assert.notCalled(instance._prefs.set);
+
+      instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {pocket: 0, tiles: [{id: 1}]}});
+      assert.notCalled(instance._prefs.set);
+    });
+    it("should clean up top story impressions", async () => {
+      instance._prefs = {get: pref => JSON.stringify({1: 1, 2: 1, 3: 1}), set: sinon.spy()};
+
+      let fetchStub = globals.sandbox.stub();
+      globals.set("fetch", fetchStub);
+      globals.set("NewTabUtils", {blockedLinks: {isBlocked: globals.sandbox.spy()}});
+
+      instance.stories_endpoint = "stories-endpoint";
+      const response = {"recommendations": [{"id": 3}, {"id": 4}, {"id": 5}]};
+      fetchStub.resolves({ok: true, status: 200, json: () => Promise.resolve(response)});
+      await instance.fetchStories();
+
+      // Should remove impressions for rec 1 and 2 as no longer in the feed
+      assert.calledWith(instance._prefs.set.firstCall, REC_IMPRESSION_TRACKING_PREF, JSON.stringify({3: 1}));
     });
   });
   describe("#spocs", () => {
     it("should insert spoc at provided interval", async () => {
       let fetchStub = globals.sandbox.stub();
       globals.set("fetch", fetchStub);
       globals.set("NewTabUtils", {blockedLinks: {isBlocked: globals.sandbox.spy()}});
 
@@ -574,22 +624,22 @@ describe("Top Stories Feed", () => {
         "settings": {"spocsPerNewTabs": 2},
         "spocs": [{"id": 1, "campaign_id": 5}, {"id": 4, "campaign_id": 6}]
       };
       fetchStub.resolves({ok: true, status: 200, json: () => Promise.resolve(response)});
       await instance.fetchStories();
 
       // simulate impressions for campaign 5 and 6
       instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {tiles: [{id: 3}, {id: 2}, {id: 1}]}});
-      instance._prefs.get = pref => JSON.stringify({5: [0]});
+      instance._prefs.get = pref => (pref === SPOC_IMPRESSION_TRACKING_PREF) && JSON.stringify({5: [0]});
       instance.onAction({type: at.TELEMETRY_IMPRESSION_STATS, data: {tiles: [{id: 3}, {id: 2}, {id: 4}]}});
 
       let expectedPrefValue = JSON.stringify({5: [0], 6: [0]});
       assert.calledWith(instance._prefs.set.secondCall, SPOC_IMPRESSION_TRACKING_PREF, expectedPrefValue);
-      instance._prefs.get = pref => expectedPrefValue;
+      instance._prefs.get = pref => (pref === SPOC_IMPRESSION_TRACKING_PREF) && expectedPrefValue;
 
       // remove campaign 5 from response
       const updatedResponse = {
         "settings": {"spocsPerNewTabs": 2},
         "spocs": [{"id": 4, "campaign_id": 6}]
       };
       fetchStub.resolves({ok: true, status: 200, json: () => Promise.resolve(updatedResponse)});
       await instance.fetchStories();
@@ -707,24 +757,39 @@ describe("Top Stories Feed", () => {
       instance.affinityProvider = undefined;
       instance.cache.set = sinon.spy();
 
       instance.observe("", "idle-daily");
       assert.isUndefined(instance.affinityProvider);
 
       instance.personalized = true;
       instance.updateSettings({timeSegments: {}, domainAffinityParameterSets: {}});
+      clock.tick(MIN_DOMAIN_AFFINITIES_UPDATE_TIME);
       instance.observe("", "idle-daily");
       assert.isDefined(instance.affinityProvider);
       assert.calledOnce(instance.cache.set);
-      assert.calledWith(instance.cache.set, "domainAffinities", instance.affinityProvider.getAffinities());
+      assert.calledWith(instance.cache.set, "domainAffinities",
+        Object.assign({}, instance.affinityProvider.getAffinities(), {"_timestamp": MIN_DOMAIN_AFFINITIES_UPDATE_TIME}));
+    });
+    it("should not update domain affinities too often", () => {
+      instance.init();
+      instance.affinityProvider = undefined;
+      instance.cache.set = sinon.spy();
+
+      instance.personalized = true;
+      instance.updateSettings({timeSegments: {}, domainAffinityParameterSets: {}});
+      clock.tick(MIN_DOMAIN_AFFINITIES_UPDATE_TIME);
+      instance.domainAffinitiesLastUpdated = Date.now();
+      instance.observe("", "idle-daily");
+      assert.isUndefined(instance.affinityProvider);
     });
     it("should send performance telemetry when updating domain affinities", () => {
       instance.init();
       instance.personalized = true;
+      clock.tick(MIN_DOMAIN_AFFINITIES_UPDATE_TIME);
       instance.updateSettings({timeSegments: {}, domainAffinityParameterSets: {}});
       instance.observe("", "idle-daily");
 
       assert.calledOnce(instance.store.dispatch);
       let action = instance.store.dispatch.firstCall.args[0];
       assert.equal(action.type, at.TELEMETRY_PERFORMANCE_EVENT);
       assert.equal(action.data.event, "topstories.domain.affinity.calculation.ms");
     });
@@ -750,16 +815,17 @@ describe("Top Stories Feed", () => {
 
       assert.deepEqual(instance.spocs, [{"url": "not_blocked"}]);
     });
     it("should reset domain affinity scores if version changed", () => {
       instance.init();
       instance.personalized = true;
       instance.resetDomainAffinityScores = sinon.spy();
       instance.updateSettings({timeSegments: {}, domainAffinityParameterSets: {}, version: "1"});
+      clock.tick(MIN_DOMAIN_AFFINITIES_UPDATE_TIME);
       instance.observe("", "idle-daily");
       assert.notCalled(instance.resetDomainAffinityScores);
 
       instance.updateSettings({timeSegments: {}, domainAffinityParameterSets: {}, version: "2"});
       assert.calledOnce(instance.resetDomainAffinityScores);
     });
   });
   describe("#loadCachedData", () => {