Backed out changeset be23dc007b53 (bug 1451050) for bustage [automation/package] Error 2 on a CLOSED TREE
authorMargareta Eliza Balazs <ebalazs@mozilla.com>
Wed, 16 May 2018 13:24:53 +0300
changeset 472695 54538505ee0a40b4eac7ca35cfbfe870118089b2
parent 472694 1a3c751f67b60c37c827d719a505edcca8144fde
child 472696 e3c0dd79cbfc5c57e65aa0f671a8c3cf446b4dfe
push id9374
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:43:20 +0000
treeherdermozilla-beta@160e085dfb0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1451050
milestone62.0a1
backs outbe23dc007b537d742e1721ac66e4f4aef1b7aa63
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset be23dc007b53 (bug 1451050) for bustage [automation/package] Error 2 on a CLOSED TREE
browser/base/content/test/static/browser_all_files_referenced.js
browser/installer/package-manifest.in
mobile/android/installer/package-manifest.in
services/blocklists/addons.json
services/blocklists/certificates.json
services/blocklists/gfx.json
services/blocklists/moz.build
services/blocklists/pins.json
services/blocklists/plugins.json
services/blocklists/readme.md
services/common/docs/RemoteSettings.rst
services/common/remote-settings.js
services/common/tests/unit/test_blocklist_clients.js
services/common/tests/unit/test_blocklist_targetapp_filter.js
services/common/tests/unit/test_remote_settings.js
services/common/tests/unit/xpcshell.ini
services/moz.build
services/settings/dumps/blocklists/addons.json
services/settings/dumps/blocklists/certificates.json
services/settings/dumps/blocklists/gfx.json
services/settings/dumps/blocklists/moz.build
services/settings/dumps/blocklists/plugins.json
services/settings/dumps/main/moz.build
services/settings/dumps/main/tippytop.json
services/settings/dumps/moz.build
services/settings/dumps/pinning/moz.build
services/settings/dumps/pinning/pins.json
services/settings/dumps/readme.md
services/settings/moz.build
--- a/browser/base/content/test/static/browser_all_files_referenced.js
+++ b/browser/base/content/test/static/browser_all_files_referenced.js
@@ -8,19 +8,18 @@
 // Slow on asan builds.
 requestLongerTimeout(5);
 
 var isDevtools = SimpleTest.harnessParameters.subsuite == "devtools";
 
 var gExceptionPaths = [
   "chrome://browser/content/defaultthemes/",
   "chrome://browser/locale/searchplugins/",
-  "resource://app/defaults/settings/blocklists/",
-  "resource://app/defaults/settings/main/",
-  "resource://app/defaults/settings/pinning/",
+  "resource://app/defaults/blocklists/",
+  "resource://app/defaults/pinning/",
   "resource://app/defaults/preferences/",
   "resource://gre/modules/commonjs/",
   "resource://gre/defaults/pref/",
 
   // These resources are referenced using relative paths from html files.
   "resource://payments/",
   "resource://normandy-content/shield-content-frame.js",
   "resource://normandy-content/shield-content-process.js",
--- a/browser/installer/package-manifest.in
+++ b/browser/installer/package-manifest.in
@@ -444,19 +444,18 @@
 
 ; [Default Preferences]
 ; All the pref files must be part of base to prevent migration bugs
 @RESPATH@/browser/@PREF_DIR@/firefox.js
 @RESPATH@/browser/@PREF_DIR@/firefox-branding.js
 @RESPATH@/greprefs.js
 @RESPATH@/defaults/autoconfig/prefcalls.js
 @RESPATH@/browser/defaults/permissions
-@RESPATH@/browser/defaults/settings/blocklists
-@RESPATH@/browser/defaults/settings/pinning
-@RESPATH@/browser/defaults/settings/main
+@RESPATH@/browser/defaults/blocklists
+@RESPATH@/browser/defaults/pinning
 
 ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
 ; Technically this is an app pref file, but we are keeping it in the original
 ; gre location for now.
 @RESPATH@/defaults/pref/channel-prefs.js
 
 ; Services (gre) prefs
 @RESPATH@/defaults/pref/services-sync.js
--- a/mobile/android/installer/package-manifest.in
+++ b/mobile/android/installer/package-manifest.in
@@ -82,18 +82,16 @@
 [xpcom]
 @BINPATH@/package-name.txt
 
 [browser]
 ; [Base Browser Files]
 @BINPATH@/application.ini
 @BINPATH@/platform.ini
 @BINPATH@/blocklist.xml
-@BINPATH@/defaults/settings/blocklists/addons.json
-@BINPATH@/defaults/settings/blocklists/certificates.json
 
 ; [Components]
 @BINPATH@/components/components.manifest
 
 ; JavaScript components
 @BINPATH@/components/ConsoleAPI.manifest
 @BINPATH@/components/ConsoleAPIStorage.js
 @BINPATH@/components/NotificationStorage.js
rename from services/settings/dumps/blocklists/addons.json
rename to services/blocklists/addons.json
rename from services/settings/dumps/blocklists/certificates.json
rename to services/blocklists/certificates.json
rename from services/settings/dumps/blocklists/gfx.json
rename to services/blocklists/gfx.json
rename from services/settings/dumps/moz.build
rename to services/blocklists/moz.build
--- a/services/settings/dumps/moz.build
+++ b/services/blocklists/moz.build
@@ -1,9 +1,18 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DIRS += [
-    'blocklists',
-    'main',
-    'pinning',
-]
+with Files('**'):
+    BUG_COMPONENT = ('Toolkit', 'Blocklisting')
+
+FINAL_TARGET_FILES.defaults.blocklists += ['addons.json',
+                                           'certificates.json',
+                                           'gfx.json',
+                                           'plugins.json']
+
+FINAL_TARGET_FILES.defaults.pinning += ['pins.json']
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'
rename from services/settings/dumps/pinning/pins.json
rename to services/blocklists/pins.json
rename from services/settings/dumps/blocklists/plugins.json
rename to services/blocklists/plugins.json
rename from services/settings/dumps/readme.md
rename to services/blocklists/readme.md
--- a/services/common/docs/RemoteSettings.rst
+++ b/services/common/docs/RemoteSettings.rst
@@ -85,28 +85,16 @@ When an entry has a file attached to it,
 
     data.filter(d => d.attachment)
         .forEach(async ({ attachment: { url, filename, size } }) => {
           if (size < OS.freeDiskSpace) {
             await downloadLocally(url, filename);
           }
         });
 
-Initial data
-------------
-
-For newly created user profiles, the list of entries returned by the ``.get()`` method will be empty until the first synchronization happens.
-
-It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet. It will thus serve as the default dataset and also reduce the amount of data to be downloaded on the first synchronization.
-
-#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder
-#. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build``
-
-Now, when ``RemoteSettings("some-key").get()`` is called from an empty profile, the ``some-key.json`` file is going to be loaded before the results are returned.
-
 
 Uptake Telemetry
 ================
 
 Some :ref:`uptake telemetry <telemetry/collection/uptake>` is collected in order to monitor how remote settings are propagated.
 
 It is submitted to a single :ref:`keyed histogram <histogram-type-keyed>` whose id is ``UPTAKE_REMOTE_CONTENT_RESULT_1`` and the keys are prefixed with ``main/`` (eg. ``main/a-key`` in the above example).
 
--- a/services/common/remote-settings.js
+++ b/services/common/remote-settings.js
@@ -208,31 +208,16 @@ class RemoteSettingsClient {
    * @param  {Object} options.order   The order to apply   (default: `-last_modified`).
    * @return {Promise}
    */
   async get(options = {}) {
     // In Bug 1451031, we will do some jexl filtering to limit the list items
     // whose target is matched.
     const { filters = {}, order } = options;
     const c = await this.openCollection();
-
-    const timestamp = await c.db.getLastModified();
-    // If the local database was never synchronized, then we attempt to load
-    // a packaged JSON dump.
-    if (timestamp == null) {
-      try {
-        const { data } = await this._loadDumpFile();
-        await c.loadDump(data);
-      } catch (e) {
-        // Report but return an empty list since there will be no data anyway.
-        Cu.reportError(e);
-        return [];
-      }
-    }
-
     const { data } = await c.list({ filters, order });
     return this._filterEntries(data);
   }
 
   /**
    * Synchronize from Kinto server, if necessary.
    *
    * @param {int}  lastModified       the lastModified date (on the server) for
@@ -412,17 +397,17 @@ class RemoteSettingsClient {
   }
 
   /**
    * Load the the JSON file distributed with the release for this collection.
    */
   async _loadDumpFile() {
     // Replace OS specific path separator by / for URI.
     const { components: folderFile } = OS.Path.split(this.filename);
-    const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
+    const fileURI = `resource://app/defaults/${folderFile.join("/")}`;
     const response = await fetch(fileURI);
     if (!response.ok) {
       throw new Error(`Could not read from '${fileURI}'`);
     }
     // Will be rejected if JSON is invalid.
     return response.json();
   }
 
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -1,23 +1,20 @@
 const { Constructor: CC } = Components;
 
-ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 
 const BlocklistClients = ChromeUtils.import("resource://services-common/blocklist-clients.js", {});
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
 
-const IS_ANDROID = AppConstants.platform == "android";
-
 
 let gBlocklistClients;
 let server;
 
 async function readJSON(filepath) {
   const binaryData = await OS.File.read(filepath);
   const textData = (new TextDecoder()).decode(binaryData);
   return Promise.resolve(JSON.parse(textData));
@@ -97,53 +94,24 @@ function run_test() {
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
 
 add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is_empty() {
-  const november2016 = 1480000000000;
-
   for (let {client} of gBlocklistClients) {
-    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
-      // On Android we don't ship the dumps of plugins and gfx.
-      continue;
-    }
-
     // Test an empty db populates, but don't reach server (specified timestamp <= dump).
     await client.maybeSync(1, Date.now());
 
     // Verify the loaded data has status to synced:
     const collection = await client.openCollection();
     const { data: list } = await collection.list();
     equal(list[0]._status, "synced");
-
-    // Verify that the internal timestamp was updated.
-    const timestamp = await collection.db.getLastModified();
-    ok(timestamp > november2016, `Loaded dump of ${client.collectionName} has timestamp ${timestamp}`);
-  }
-});
-add_task(clear_state);
-
-add_task(async function test_initial_dump_is_loaded_when_using_get_on_empty_collection() {
-  for (let {client} of gBlocklistClients) {
-    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
-      // On Android we don't ship the dumps of plugins and gfx.
-      continue;
-    }
-    // Internal database is empty.
-    const collection = await client.openCollection();
-    const { data: list } = await collection.list();
-    equal(list.length, 0);
-
-    // Calling .get() will load the dump.
-    const afterLoaded = await client.get();
-    ok(afterLoaded.length > 0, `Loaded dump of ${client.collectionName} has ${afterLoaded.length} records`);
   }
 });
 add_task(clear_state);
 
 add_task(async function test_list_is_written_to_file_in_profile() {
   for (let {client, testData} of gBlocklistClients) {
     const filePath = OS.Path.join(OS.Constants.Path.profileDir, client.filename);
     const profFile = new FileUtils.File(filePath);
--- a/services/common/tests/unit/test_blocklist_targetapp_filter.js
+++ b/services/common/tests/unit/test_blocklist_targetapp_filter.js
@@ -11,17 +11,16 @@ async function clear_state() {
   await collection.clear();
 }
 
 async function createRecords(records) {
   const collection = await client.openCollection();
   for (const record of records) {
     await collection.create(record);
   }
-  collection.db.saveLastModified(42); // Simulate sync (and prevent load dump).
 }
 
 
 function run_test() {
   // This will initialize the remote settings clients for blocklists,
   // with their specific options etc.
   BlocklistClients.initialize();
   // Obtain one of the instantiated client for our tests.
--- a/services/common/tests/unit/test_remote_settings.js
+++ b/services/common/tests/unit/test_remote_settings.js
@@ -98,27 +98,16 @@ add_task(async function test_records_cha
 
   await client.maybeSync(2000, Date.now());
 
   const data = await client.get();
   equal(data[0].website, "https://some-website.com");
 });
 add_task(clear_state);
 
-add_task(async function test_default_records_come_from_a_local_dump_when_database_is_empty() {
-  // When collection is unknown, no dump is loaded, and there is no error.
-  let data = await RemoteSettings("some-unknown-key").get();
-  equal(data.length, 0);
-
-  // When collection has a dump in services/settings/dumps/{bucket}/{collection}.json
-  data = await RemoteSettings("certificates", { bucketName: "blocklists" }).get();
-  notEqual(data.length, 0);
-});
-add_task(clear_state);
-
 add_task(async function test_sync_event_provides_information_about_records() {
   const serverTime = Date.now();
 
   let eventData;
   client.on("sync", ({ data }) => eventData = data);
 
   await client.maybeSync(2000, serverTime - 1000);
   equal(eventData.current.length, 1);
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@@ -4,18 +4,18 @@ firefox-appdir = browser
 support-files =
   test_storage_adapter/**
   test_blocklist_signatures/**
 
 # Test load modules first so syntax failures are caught early.
 [test_load_modules.js]
 
 [test_blocklist_certificates.js]
-# Skip signature tests for Thunderbird (Bug 1341983).
-skip-if = appname == "thunderbird"
+# Initial JSON data for blocklists are not shipped on Android.
+skip-if = (os == "android" || appname == "thunderbird")
 tags = blocklist
 [test_blocklist_clients.js]
 tags = blocklist
 [test_blocklist_targetapp_filter.js]
 tags = blocklist
 [test_blocklist_pinning.js]
 tags = blocklist
 [test_remote_settings.js]
--- a/services/moz.build
+++ b/services/moz.build
@@ -5,18 +5,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files('moz.build'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
 DIRS += [
     'common',
     'crypto',
-    'settings',
 ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
     DIRS += [
         'fxaccounts',
+        'blocklists',
     ]
 
 if CONFIG['MOZ_SERVICES_SYNC']:
     DIRS += ['sync']
deleted file mode 100644
--- a/services/settings/dumps/blocklists/moz.build
+++ /dev/null
@@ -1,16 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-with Files('**'):
-    BUG_COMPONENT = ('Toolkit', 'Blocklisting')
-
-FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons.json',
-                                                    'certificates.json',
-                                                    'gfx.json',
-                                                    'plugins.json']
-
-if CONFIG['MOZ_BUILD_APP'] == 'browser':
-    DIST_SUBDIR = 'browser'
deleted file mode 100644
--- a/services/settings/dumps/main/moz.build
+++ /dev/null
@@ -1,10 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-FINAL_TARGET_FILES.defaults.settings.main += [
-    'tippytop.json',
-]
-
-if CONFIG['MOZ_BUILD_APP'] == 'browser':
-    DIST_SUBDIR = 'browser'
deleted file mode 100644
--- a/services/settings/dumps/main/tippytop.json
+++ /dev/null
@@ -1,1 +0,0 @@
-{"data":[]}
\ No newline at end of file
deleted file mode 100644
--- a/services/settings/dumps/pinning/moz.build
+++ /dev/null
@@ -1,8 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-FINAL_TARGET_FILES.defaults.settings.pinning += ['pins.json']
-
-if CONFIG['MOZ_BUILD_APP'] == 'browser':
-    DIST_SUBDIR = 'browser'
deleted file mode 100644
--- a/services/settings/moz.build
+++ /dev/null
@@ -1,10 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-with Files('**'):
-    BUG_COMPONENT = ('Firefox', 'Remote Settings Client')
-
-DIRS += [
-    'dumps',
-]