Bug 1573930 - FF prefs to control DS and mitigation strat r=Mardak
authorScott <scott.downe@gmail.com>
Thu, 15 Aug 2019 13:01:26 +0000
changeset 488321 c5f74c51b12fd2fda0a6e0e2fd4098ce72a74228
parent 488320 8c98f638d6cd56e34bd1cdb5f7a70da70b5ea199
child 488322 956ab93a93c7ab130819d6d7c9a3fb05ec6093fa
push id36440
push userncsoregi@mozilla.com
push dateFri, 16 Aug 2019 03:57:48 +0000
treeherdermozilla-central@a58b7dc85887 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersMardak
bugs1573930
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1573930 - FF prefs to control DS and mitigation strat r=Mardak Differential Revision: https://phabricator.services.mozilla.com/D42064
browser/app/profile/firefox.js
browser/components/newtab/lib/DiscoveryStreamFeed.jsm
browser/components/newtab/lib/PrefsFeed.jsm
browser/components/newtab/lib/TopStoriesFeed.jsm
browser/components/newtab/test/browser/browser.ini
browser/components/newtab/test/unit/lib/DiscoveryStreamFeed.test.js
browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1305,16 +1305,25 @@ pref("browser.newtabpage.activity-stream
 
 // ASRouter provider configuration
 pref("browser.newtabpage.activity-stream.asrouter.providers.cfr", "{\"id\":\"cfr\",\"enabled\":true,\"type\":\"remote-settings\",\"bucket\":\"cfr\",\"frequency\":{\"custom\":[{\"period\":\"daily\",\"cap\":1}]},\"categories\":[\"cfrAddons\",\"cfrFeatures\"],\"updateCycleInMs\":3600000}");
 // This url, if changed, MUST continue to point to an https url. Pulling arbitrary content to inject into
 // this page over http opens us up to a man-in-the-middle attack that we'd rather not face. If you are a downstream
 // repackager of this code using an alternate snippet url, please keep your users safe
 pref("browser.newtabpage.activity-stream.asrouter.providers.snippets", "{\"id\":\"snippets\",\"enabled\":true,\"type\":\"remote\",\"url\":\"https://snippets.cdn.mozilla.net/%STARTPAGE_VERSION%/%NAME%/%VERSION%/%APPBUILDID%/%BUILD_TARGET%/%LOCALE%/%CHANNEL%/%OS_VERSION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%/\",\"updateCycleInMs\":14400000}");
 
+// These prefs control if Discovery Stream is enabled.
+#ifdef NIGHTLY_BUILD
+pref("browser.newtabpage.activity-stream.discoverystream.enabled", true);
+#else
+pref("browser.newtabpage.activity-stream.discoverystream.enabled", false);
+#endif
+pref("browser.newtabpage.activity-stream.discoverystream.hardcoded-basic-layout", false);
+pref("browser.newtabpage.activity-stream.discoverystream.spocs-endpoint", "");
+
 // The pref controls if search hand-off is enabled for Activity Stream.
 #ifdef NIGHTLY_BUILD
 pref("browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar", true);
 #else
 pref("browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar", false);
 #endif
 
 pref("trailhead.firstrun.branches", "join-supercharge");
--- a/browser/components/newtab/lib/DiscoveryStreamFeed.jsm
+++ b/browser/components/newtab/lib/DiscoveryStreamFeed.jsm
@@ -50,23 +50,27 @@ const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 
 const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
 const MIN_DOMAIN_AFFINITIES_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours
 const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
 const DEFAULT_MAX_HISTORY_QUERY_RESULTS = 1000;
 const FETCH_TIMEOUT = 45 * 1000;
 const PREF_CONFIG = "discoverystream.config";
 const PREF_ENDPOINTS = "discoverystream.endpoints";
 const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId";
+const PREF_ENABLED = "discoverystream.enabled";
+const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout";
+const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint";
 const PREF_TOPSTORIES = "feeds.section.topstories";
 const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear";
 const PREF_SHOW_SPONSORED = "showSponsored";
 const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions";
 const PREF_REC_IMPRESSIONS = "discoverystream.rec.impressions";
 
 let defaultLayoutResp;
+let basicLayoutResp;
 
 this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
   constructor() {
     // Internal state for checking if we've intialized all our data
     this.loaded = false;
 
     // Persistent cache for remote endpoint data.
     this.cache = new PersistentCache(CACHE_KEY, true);
@@ -142,16 +146,20 @@ this.DiscoveryStreamFeed = class Discove
     } catch (e) {
       // istanbul ignore next
       this._prefCache.config = {};
       // istanbul ignore next
       Cu.reportError(
         `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config. ${e}`
       );
     }
+    this._prefCache.config.enabled =
+      this._prefCache.config.enabled &&
+      this.store.getState().Prefs.values[PREF_ENABLED];
+
     return this._prefCache.config;
   }
 
   resetConfigDefauts() {
     this.store.dispatch({
       type: at.CLEAR_PREF,
       data: {
         name: PREF_CONFIG,
@@ -325,17 +333,34 @@ this.DiscoveryStreamFeed = class Discove
 
   async loadLayout(sendUpdate, isStartup) {
     let layout = {};
     if (!this.config.hardcoded_layout) {
       layout = await this.fetchLayout(isStartup);
     }
 
     if (!layout || !layout.layout) {
-      layout = { lastUpdate: Date.now(), ...defaultLayoutResp };
+      if (
+        this.config.hardcoded_basic_layout ||
+        this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT]
+      ) {
+        layout = { lastUpdate: Date.now(), ...basicLayoutResp };
+      } else {
+        layout = { lastUpdate: Date.now(), ...defaultLayoutResp };
+      }
+    }
+
+    if (
+      layout.spocs &&
+      (this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] ||
+        this.config.spocs_endpoint)
+    ) {
+      layout.spocs.url =
+        this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] ||
+        this.config.spocs_endpoint;
     }
 
     sendUpdate({
       type: at.DISCOVERY_STREAM_LAYOUT_UPDATE,
       data: layout,
     });
     if (
       layout.spocs &&
@@ -1233,16 +1258,19 @@ this.DiscoveryStreamFeed = class Discove
         break;
       case at.UNINIT:
         // When this feed is shutting down:
         this.uninitPrefs();
         break;
       case at.PREF_CHANGED:
         switch (action.data.name) {
           case PREF_CONFIG:
+          case PREF_ENABLED:
+          case PREF_HARDCODED_BASIC_LAYOUT:
+          case PREF_SPOCS_ENDPOINT:
             // Clear the cached config and broadcast the newly computed value
             this._prefCache.config = null;
             this.store.dispatch(
               ac.BroadcastToContent({
                 type: at.DISCOVERY_STREAM_CONFIG_CHANGE,
                 data: this.config,
               })
             );
@@ -1382,9 +1410,101 @@ defaultLayoutResp = {
             ".ds-navigation": "margin-top: -10px;",
           },
         },
       ],
     },
   ],
 };
 
+// Hardcoded version of layout_variant `basic`
+basicLayoutResp = {
+  spocs: {
+    url: "https://spocs.getpocket.com/spocs",
+    spocs_per_domain: 1,
+  },
+  layout: [
+    {
+      width: 12,
+      components: [
+        {
+          type: "TopSites",
+          header: {
+            title: "Top Sites",
+          },
+          properties: {},
+        },
+        {
+          type: "Message",
+          header: {
+            title: "Recommended by Pocket",
+            subtitle: "",
+            link_text: "How it works",
+            link_url: "https://getpocket.com/firefox/new_tab_learn_more",
+            icon:
+              "resource://activity-stream/data/content/assets/glyph-pocket-16.svg",
+          },
+          properties: {},
+          styles: {
+            ".ds-message": "margin-bottom: -20px",
+          },
+        },
+        {
+          type: "CardGrid",
+          properties: {
+            items: 3,
+          },
+          header: {
+            title: "",
+          },
+          feed: {
+            embed_reference: null,
+            url:
+              "https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=en-US&feed_variant=default_spocs_on",
+          },
+          spocs: {
+            probability: 1,
+            positions: [
+              {
+                index: 2,
+              },
+            ],
+          },
+        },
+        {
+          type: "Navigation",
+          properties: {
+            alignment: "left-align",
+            links: [
+              {
+                name: "Must Reads",
+                url: "https://getpocket.com/explore/must-reads?src=fx_new_tab",
+              },
+              {
+                name: "Productivity",
+                url:
+                  "https://getpocket.com/explore/productivity?src=fx_new_tab",
+              },
+              {
+                name: "Health",
+                url: "https://getpocket.com/explore/health?src=fx_new_tab",
+              },
+              {
+                name: "Finance",
+                url: "https://getpocket.com/explore/finance?src=fx_new_tab",
+              },
+              {
+                name: "Technology",
+                url: "https://getpocket.com/explore/technology?src=fx_new_tab",
+              },
+              {
+                name: "More Recommendations ›",
+                url: "https://getpocket.com/explore/trending?src=fx_new_tab",
+              },
+            ],
+          },
+        },
+      ],
+    },
+  ],
+};
+
 const EXPORTED_SYMBOLS = ["DiscoveryStreamFeed"];
--- a/browser/components/newtab/lib/PrefsFeed.jsm
+++ b/browser/components/newtab/lib/PrefsFeed.jsm
@@ -79,16 +79,43 @@ this.PrefsFeed = class PrefsFeed {
     let handoffToAwesomebarPrefValue = Services.prefs.getBoolPref(
       "browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar"
     );
     values["improvesearch.handoffToAwesomebar"] = handoffToAwesomebarPrefValue;
     this._prefMap.set("improvesearch.handoffToAwesomebar", {
       value: handoffToAwesomebarPrefValue,
     });
 
+    let discoveryStreamEnabled = Services.prefs.getBoolPref(
+      "browser.newtabpage.activity-stream.discoverystream.enabled",
+      false
+    );
+    let discoveryStreamHardcodedBasicLayout = Services.prefs.getBoolPref(
+      "browser.newtabpage.activity-stream.discoverystream.hardcoded-basic-layout",
+      false
+    );
+    let discoveryStreamSpocsEndpoint = Services.prefs.getStringPref(
+      "browser.newtabpage.activity-stream.discoverystream.spocs-endpoint",
+      ""
+    );
+    values["discoverystream.enabled"] = discoveryStreamEnabled;
+    this._prefMap.set("discoverystream.enabled", {
+      value: discoveryStreamEnabled,
+    });
+    values[
+      "discoverystream.hardcoded-basic-layout"
+    ] = discoveryStreamHardcodedBasicLayout;
+    this._prefMap.set("discoverystream.hardcoded-basic-layout", {
+      value: discoveryStreamHardcodedBasicLayout,
+    });
+    values["discoverystream.spocs-endpoint"] = discoveryStreamSpocsEndpoint;
+    this._prefMap.set("discoverystream.spocs-endpoint", {
+      value: discoveryStreamSpocsEndpoint,
+    });
+
     // Set the initial state of all prefs in redux
     this.store.dispatch(
       ac.BroadcastToContent({ type: at.PREFS_INITIAL_VALUES, data: values })
     );
   }
 
   removeListeners() {
     this._prefs.ignoreBranch(this);
--- a/browser/components/newtab/lib/TopStoriesFeed.jsm
+++ b/browser/components/newtab/lib/TopStoriesFeed.jsm
@@ -49,28 +49,34 @@ const STORIES_UPDATE_TIME = 30 * 60 * 10
 const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours
 const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours
 const MIN_DOMAIN_AFFINITIES_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours
 const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
 const SECTION_ID = "topstories";
 const IMPRESSION_SOURCE = "TOP_STORIES";
 const SPOC_IMPRESSION_TRACKING_PREF =
   "feeds.section.topstories.spoc.impressions";
+const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled";
+const DISCOVERY_STREAM_PREF_ENABLED_PATH =
+  "browser.newtabpage.activity-stream.discoverystream.enabled";
 const REC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.rec.impressions";
 const OPTIONS_PREF = "feeds.section.topstories.options";
 const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
 const DISCOVERY_STREAM_PREF = "discoverystream.config";
 
 this.TopStoriesFeed = class TopStoriesFeed {
   constructor(ds) {
     // Use discoverystream config pref default values for fast path and
     // if needed lazy load activity stream top stories feed based on
     // actual user preference when INIT and PREF_CHANGED is invoked
     this.discoveryStreamEnabled =
-      ds && ds.value && JSON.parse(ds.value).enabled;
+      ds &&
+      ds.value &&
+      JSON.parse(ds.value).enabled &&
+      Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false);
     if (!this.discoveryStreamEnabled) {
       this.initializeProperties();
     }
   }
 
   initializeProperties() {
     this.contentUpdateQueue = [];
     this.spocCampaignMap = new Map();
@@ -779,17 +785,19 @@ this.TopStoriesFeed = class TopStoriesFe
 
   lazyLoadTopStories(dsPref) {
     let _dsPref = dsPref;
     if (!_dsPref) {
       _dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF];
     }
 
     try {
-      this.discoveryStreamEnabled = JSON.parse(_dsPref).enabled;
+      this.discoveryStreamEnabled =
+        JSON.parse(_dsPref).enabled &&
+        this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED];
     } catch (e) {
       // Load activity stream top stories if fail to determine discovery stream state
       this.discoveryStreamEnabled = false;
     }
 
     // Return without invoking initialization if top stories are loaded
     if (this.storiesLoaded) {
       return;
@@ -805,16 +813,19 @@ this.TopStoriesFeed = class TopStoriesFe
     switch (action.type) {
       case at.INIT:
         this.lazyLoadTopStories();
         break;
       case at.PREF_CHANGED:
         if (action.data.name === DISCOVERY_STREAM_PREF) {
           this.lazyLoadTopStories(action.data.value);
         }
+        if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) {
+          this.lazyLoadTopStories();
+        }
         break;
       case at.UNINIT:
         this.uninit();
         break;
     }
   }
 
   async onAction(action) {
--- a/browser/components/newtab/test/browser/browser.ini
+++ b/browser/components/newtab/test/browser/browser.ini
@@ -1,15 +1,16 @@
 [DEFAULT]
 support-files =
   blue_page.html
   red_page.html
   head.js
 prefs =
   browser.newtabpage.activity-stream.debug=false
+  browser.newtabpage.activity-stream.discoverystream.enabled=true
   browser.newtabpage.activity-stream.discoverystream.endpoints=data:
   browser.newtabpage.activity-stream.feeds.section.topstories=true
   browser.newtabpage.activity-stream.feeds.section.topstories.options={"provider_name":""}
 
 [browser_aboutwelcome.js]
 [browser_as_load_location.js]
 [browser_as_render.js]
 [browser_asrouter_snippets.js]
--- a/browser/components/newtab/test/unit/lib/DiscoveryStreamFeed.test.js
+++ b/browser/components/newtab/test/unit/lib/DiscoveryStreamFeed.test.js
@@ -74,27 +74,33 @@ describe("DiscoveryStreamFeed", () => {
       "lib/UserDomainAffinityProvider.jsm": {
         UserDomainAffinityProvider: FakeUserDomainAffinityProvider,
       },
     }));
 
     globals = new GlobalOverrider();
     globals.set("gUUIDGenerator", { generateUUID: () => FAKE_UUID });
 
+    sandbox
+      .stub(global.Services.prefs, "getBoolPref")
+      .withArgs("browser.newtabpage.activity-stream.discoverystream.enabled")
+      .returns(true);
+
     // Feed
     feed = new DiscoveryStreamFeed();
     feed.store = createStore(combineReducers(reducers), {
       Prefs: {
         values: {
           [CONFIG_PREF_NAME]: JSON.stringify({
             enabled: false,
             show_spocs: false,
             layout_endpoint: DUMMY_ENDPOINT,
           }),
           [ENDPOINTS_PREF_NAME]: DUMMY_ENDPOINT,
+          "discoverystream.enabled": true,
         },
       },
     });
     global.fetch.resetHistory();
 
     sandbox.stub(feed, "_maybeUpdateCachedData").resolves();
 
     globals.set("setTimeout", callback => {
@@ -289,16 +295,94 @@ describe("DiscoveryStreamFeed", () => {
       await feed.loadLayout(feed.store.dispatch);
 
       assert.notCalled(feed.fetchLayout);
       assert.equal(
         feed.store.getState().DiscoveryStream.spocs.spocs_endpoint,
         "https://spocs.getpocket.com/spocs"
       );
     });
+    it("should use local basic layout with hardcoded_layout and hardcoded_basic_layout being true", async () => {
+      feed.config.hardcoded_layout = true;
+      feed.config.hardcoded_basic_layout = true;
+      sandbox.stub(feed, "fetchLayout").returns(Promise.resolve(""));
+
+      await feed.loadLayout(feed.store.dispatch);
+
+      assert.notCalled(feed.fetchLayout);
+      assert.equal(
+        feed.store.getState().DiscoveryStream.spocs.spocs_endpoint,
+        "https://spocs.getpocket.com/spocs"
+      );
+      const { layout } = feed.store.getState().DiscoveryStream;
+      assert.equal(layout[0].components[2].properties.items, 3);
+    });
+    it("should use new spocs endpoint if in the config", async () => {
+      feed.config.spocs_endpoint = "https://spocs.getpocket.com/spocs2";
+
+      await feed.loadLayout(feed.store.dispatch);
+
+      assert.equal(
+        feed.store.getState().DiscoveryStream.spocs.spocs_endpoint,
+        "https://spocs.getpocket.com/spocs2"
+      );
+    });
+    it("should use local basic layout with hardcoded_layout and FF pref hardcoded_basic_layout", async () => {
+      feed.config.hardcoded_layout = true;
+      feed.store = createStore(combineReducers(reducers), {
+        Prefs: {
+          values: {
+            [CONFIG_PREF_NAME]: JSON.stringify({
+              enabled: false,
+              show_spocs: false,
+              layout_endpoint: DUMMY_ENDPOINT,
+            }),
+            [ENDPOINTS_PREF_NAME]: DUMMY_ENDPOINT,
+            "discoverystream.enabled": true,
+            "discoverystream.hardcoded-basic-layout": true,
+          },
+        },
+      });
+
+      sandbox.stub(feed, "fetchLayout").returns(Promise.resolve(""));
+
+      await feed.loadLayout(feed.store.dispatch);
+
+      assert.notCalled(feed.fetchLayout);
+      assert.equal(
+        feed.store.getState().DiscoveryStream.spocs.spocs_endpoint,
+        "https://spocs.getpocket.com/spocs"
+      );
+      const { layout } = feed.store.getState().DiscoveryStream;
+      assert.equal(layout[0].components[2].properties.items, 3);
+    });
+    it("should use new spocs endpoint if in a FF pref", async () => {
+      feed.store = createStore(combineReducers(reducers), {
+        Prefs: {
+          values: {
+            [CONFIG_PREF_NAME]: JSON.stringify({
+              enabled: false,
+              show_spocs: false,
+              layout_endpoint: DUMMY_ENDPOINT,
+            }),
+            [ENDPOINTS_PREF_NAME]: DUMMY_ENDPOINT,
+            "discoverystream.enabled": true,
+            "discoverystream.spocs-endpoint":
+              "https://spocs.getpocket.com/spocs2",
+          },
+        },
+      });
+
+      await feed.loadLayout(feed.store.dispatch);
+
+      assert.equal(
+        feed.store.getState().DiscoveryStream.spocs.spocs_endpoint,
+        "https://spocs.getpocket.com/spocs2"
+      );
+    });
     it("should fetch local layout for invalid layout endpoint or when fetch layout fails", async () => {
       feed.config.hardcoded_layout = false;
       fetchStub.resolves({ ok: false });
 
       await feed.loadLayout(feed.store.dispatch, true);
 
       assert.calledOnce(fetchStub);
       assert.equal(
--- a/browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js
+++ b/browser/components/newtab/test/unit/lib/TopStoriesFeed.test.js
@@ -166,19 +166,21 @@ describe("Top Stories Feed", () => {
     });
     it("should handle limited actions when discoverystream is enabled", async () => {
       sinon.spy(instance, "handleDisabled");
       sinon.stub(instance, "getPocketState");
       instance.store.getState = () => ({
         Prefs: {
           values: {
             "discoverystream.config": JSON.stringify({ enabled: true }),
+            "discoverystream.enabled": true,
           },
         },
       });
+
       instance.onAction({ type: at.INIT, data: {} });
 
       assert.calledOnce(instance.handleDisabled);
       instance.onAction({
         type: at.NEW_TAB_REHYDRATED,
         meta: { fromTarget: {} },
       });
       assert.notCalled(instance.getPocketState);
@@ -211,16 +213,24 @@ describe("Top Stories Feed", () => {
     it("should fire init on PREF_CHANGED", () => {
       sinon.stub(instance, "onInit");
       instance.onAction({
         type: at.PREF_CHANGED,
         data: { name: "discoverystream.config", value: {} },
       });
       assert.calledOnce(instance.onInit);
     });
+    it("should fire init on DISCOVERY_STREAM_PREF_ENABLED", () => {
+      sinon.stub(instance, "onInit");
+      instance.onAction({
+        type: at.PREF_CHANGED,
+        data: { name: "discoverystream.enabled", value: true },
+      });
+      assert.calledOnce(instance.onInit);
+    });
     it("should not fire init on PREF_CHANGED if stories are loaded", () => {
       sinon.stub(instance, "onInit");
       sinon.spy(instance, "lazyLoadTopStories");
       instance.storiesLoaded = true;
       instance.onAction({
         type: at.PREF_CHANGED,
         data: { name: "discoverystream.config", value: {} },
       });