Bug 1563226 - Download the Public Suffix List using Remote Settings r=leplatrem
authorArpit Bharti <arpitbharti73@gmail.com>
Thu, 22 Aug 2019 14:09:42 +0000
changeset 489844 3c352981d6c4796ae741dd1397a1236477d02e06
parent 489843 bad1d56f5aa8722d3e9793703679e02cbfa6fd8c
child 489845 d9b024555ac15ecc7f9e9e6e01f92a752deb9040
push id36491
push usermalexandru@mozilla.com
push dateMon, 26 Aug 2019 22:30:36 +0000
treeherdermozilla-central@5c7635de0cb6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersleplatrem
bugs1563226
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1563226 - Download the Public Suffix List using Remote Settings r=leplatrem Differential Revision: https://phabricator.services.mozilla.com/D42469
browser/components/BrowserGlue.jsm
netwerk/dns/PublicSuffixList.jsm
netwerk/dns/moz.build
netwerk/dns/tests/moz.build
netwerk/dns/tests/unit/data/fake_public_suffix_list.dat
netwerk/dns/tests/unit/data/moz.build
netwerk/dns/tests/unit/moz.build
netwerk/dns/tests/unit/test_PublicSuffixList.js
netwerk/dns/tests/unit/xpcshell.ini
--- a/browser/components/BrowserGlue.jsm
+++ b/browser/components/BrowserGlue.jsm
@@ -529,16 +529,17 @@ XPCOMUtils.defineLazyModuleGetters(this,
   PdfJs: "resource://pdf.js/PdfJs.jsm",
   PermissionUI: "resource:///modules/PermissionUI.jsm",
   PingCentre: "resource:///modules/PingCentre.jsm",
   PlacesBackups: "resource://gre/modules/PlacesBackups.jsm",
   PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
   PluralForm: "resource://gre/modules/PluralForm.jsm",
   PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.jsm",
   ProcessHangMonitor: "resource:///modules/ProcessHangMonitor.jsm",
+  PublicSuffixList: "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm",
   RemoteSettings: "resource://services-settings/remote-settings.js",
   RemoteSecuritySettings:
     "resource://gre/modules/psm/RemoteSecuritySettings.jsm",
   RFPHelper: "resource://gre/modules/RFPHelper.jsm",
   SafeBrowsing: "resource://gre/modules/SafeBrowsing.jsm",
   Sanitizer: "resource:///modules/Sanitizer.jsm",
   SaveToPocket: "chrome://pocket/content/SaveToPocket.jsm",
   SearchTelemetry: "resource:///modules/SearchTelemetry.jsm",
@@ -2201,16 +2202,20 @@ BrowserGlue.prototype = {
       this._gmpInstallManager.simpleCheckAndInstall().catch(() => {});
     });
 
     Services.tm.idleDispatchToMainThread(() => {
       RemoteSettings.init();
     });
 
     Services.tm.idleDispatchToMainThread(() => {
+      PublicSuffixList.init();
+    });
+
+    Services.tm.idleDispatchToMainThread(() => {
       RemoteSecuritySettings.init();
     });
   },
 
   _onQuitRequest: function BG__onQuitRequest(aCancelQuit, aQuitType) {
     // If user has already dismissed quit request, then do nothing
     if (aCancelQuit instanceof Ci.nsISupportsPRBool && aCancelQuit.data) {
       return;
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/PublicSuffixList.jsm
@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { RemoteSettings } = ChromeUtils.import(
+  "resource://services-settings/remote-settings.js"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const FileUtils = ChromeUtils.import("resource://gre/modules/FileUtils.jsm")
+  .FileUtils;
+
+const EXPORTED_SYMBOLS = ["PublicSuffixList"];
+
+const RECORD_ID = "tld-dafsa";
+const SIGNAL = "public-suffix-list-updated";
+
+const PublicSuffixList = {
+  CLIENT: RemoteSettings("public-suffix-list"),
+
+  init() {
+    this.CLIENT.on("sync", this.onUpdate.bind(this));
+    /* We have a single record for this collection. Let's see if we already have it locally.
+     * Note that on startup, we don't need to synchronize immediately on new profiles.
+     */
+    this.CLIENT.get({ syncIfEmpty: false, filters: { id: RECORD_ID } })
+      .then(async records => {
+        if (records.length == 1) {
+          const fileURI = await this.CLIENT.attachments.download(records[0]);
+          // Send a signal so that the C++ code loads the updated list on startup.
+          this.notifyUpdate(fileURI);
+        }
+      })
+      .catch(err => console.error(err));
+  },
+
+  /**
+   * This method returns the path to the file based on the file uri received
+   * Example:
+   * On windows "file://C:/Users/AppData/main/public-suffix-list/dafsa.bin"
+   * will be converted to "C:\\Users\\main\\public-suffix-list\\dafsa.bin
+   *
+   * On macOS/linux "file:///home/main/public-suffix-list/dafsa.bin"
+   * will be converted to "/home/main/public-suffix-list/dafsa.bin"
+   */
+  getFilePath(fileURI) {
+    const uri = Services.io.newURI(fileURI);
+    const file = uri.QueryInterface(Ci.nsIFileURL).file;
+    return file.path;
+  },
+
+  notifyUpdate(fileURI) {
+    const filePath = this.getFilePath(fileURI);
+    const nsifile = new FileUtils.File(filePath);
+    /* Send a signal to be read by the C++, the method
+     * ::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData)
+     * at netwerk/dns/nsEffectiveTLDService.cpp
+     */
+    Services.obs.notifyObservers(
+      nsifile, // aSubject
+      SIGNAL, // aTopic
+      filePath // aData
+    );
+  },
+
+  async onUpdate({ data: { created, updated, deleted } }) {
+    // In theory, this will never happen, we will never delete the record.
+    if (deleted.length == 1) {
+      await this.CLIENT.attachments.delete(deleted[0]);
+    }
+    // Handle creation and update the same way
+    const changed = created.concat(updated.map(u => u.new));
+    /* In theory, we should never have more than one record. And if we receive
+     * this event, it's because the single record was updated.
+     */
+    if (changed.length != 1) {
+      console.warn("Unsupported sync event for Public Suffix List");
+      return;
+    }
+    // Download the updated file.
+    const fileURI = await this.CLIENT.attachments.download(changed[0]);
+    // Notify the C++ part to reload it from disk.
+    this.notifyUpdate(fileURI);
+  },
+};
--- a/netwerk/dns/moz.build
+++ b/netwerk/dns/moz.build
@@ -4,30 +4,37 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files('**'):
     BUG_COMPONENT = ('Core', 'Networking: DNS')
 
 DIRS += [
     'mdns',
+    'tests'
 ]
 
 XPIDL_SOURCES += [
     'nsIDNSByTypeRecord.idl',
     'nsIDNSListener.idl',
     'nsIDNSRecord.idl',
     'nsIDNSService.idl',
     'nsIEffectiveTLDService.idl',
     'nsIIDNService.idl',
     'nsPIDNSService.idl',
 ]
 
 XPIDL_MODULE = 'necko_dns'
 
+EXTRA_JS_MODULES['netwerk-dns'] += [
+    'PublicSuffixList.jsm',
+]
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
+
 EXPORTS += [
     'nsEffectiveTLDService.h',
 ]
 
 EXPORTS.mozilla.net += [
     'ChildDNSService.h',
     'DNS.h',
     'DNSListenerProxy.h',
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/moz.build
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'unit'
+]
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/unit/data/fake_public_suffix_list.dat
@@ -0,0 +1,57 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// This is a fake suffix list created only for the purposes of testing,
+// The real PSL is rather large thus this file is kept small by cutting out most of the suffixes
+
+// The Original list can be found at https://publicsuffix.org/list/public_suffix_list.dat,
+// Learn more about the PSL at https://publicsuffix.org.
+
+// .xpcshelltest is the fake domain created specifically for the the tests while
+// the others are ripped from the real list, serving as examples.
+
+// This fake public suffix list was used to create the binary file fake_remote_dafsa.bin
+// The binary is built at compile time and can be found at
+// obj-dir/_tests/xpcshell/netwerk/dns/tests/unit/data/
+
+// The list created with help of netwerk/dns/prepare_tlds.py and xpcom/ds/tools/make_dafsa.py
+// The build directive for the binary is at moz.build at netwerk/dns/tests/unit/data/
+
+
+
+// ===BEGIN ICANN DOMAINS===
+
+// xpcshelltest : Used in tests
+xpcshelltest
+website.xpcshelltest
+com.xpcshelltest
+edu.xpcshelltest
+gov.xpcshelltest
+net.xpcshelltest
+mil.xpcshelltest
+org.xpcshelltest
+
+// ac : https://en.wikipedia.org/wiki/.ac
+ac
+coc.ac
+com.ac
+edu.ac
+gov.ac
+net.ac
+mil.ac
+org.ac
+
+// bj : https://en.wikipedia.org/wiki/.bj
+bj
+asso.bj
+barreau.bj
+gouv.bj
+
+// bm : http://www.bermudanic.bm/dnr-text.txt
+bm
+com.bm
+edu.bm
+gov.bm
+net.bm
+org.bm
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/unit/data/moz.build
@@ -0,0 +1,14 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+GENERATED_FILES = ['fake_remote_dafsa.bin']
+
+fake_remote_dafsa = GENERATED_FILES['fake_remote_dafsa.bin']
+fake_remote_dafsa.script = '../../../prepare_tlds.py'
+fake_remote_dafsa.inputs = ['fake_public_suffix_list.dat']
+fake_remote_dafsa.flags = ['bin']
+
+TEST_HARNESS_FILES.xpcshell.netwerk.dns.tests.unit.data += ['!fake_remote_dafsa.bin']
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/unit/moz.build
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'data'
+]
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/unit/test_PublicSuffixList.js
@@ -0,0 +1,167 @@
+"use strict";
+
+const { PublicSuffixList } = ChromeUtils.import(
+  "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm"
+);
+const { TestUtils } = ChromeUtils.import(
+  "resource://testing-common/TestUtils.jsm"
+);
+
+const CLIENT = PublicSuffixList.CLIENT;
+const SIGNAL = "public-suffix-list-updated";
+
+const PAYLOAD_UPDATED_RECORDS = {
+  current: [{ id: "tld-dafsa", "commit-hash": "current-commit-hash" }],
+  created: [],
+  updated: [
+    {
+      old: { id: "tld-dafsa", "commit-hash": "current-commit-hash" },
+      new: { id: "tld-dafsa", "commit-hash": "new-commit-hash" },
+    },
+  ],
+  deleted: [],
+};
+const PAYLOAD_CREATED_RECORDS = {
+  current: [],
+  created: [
+    {
+      id: "tld-dafsa",
+      "commit-hash": "new-commit-hash",
+      attachment: {},
+    },
+  ],
+  updated: [],
+  deleted: [],
+};
+const PAYLOAD_UPDATED_AND_CREATED_RECORDS = {
+  current: [{ id: "tld-dafsa", "commit-hash": "current-commit-hash" }],
+  created: [{ id: "tld-dafsa", "commit-hash": "another-commit-hash" }],
+  updated: [
+    {
+      old: { id: "tld-dafsa", "commit-hash": "current-commit-hash" },
+      new: { id: "tld-dafsa", "commit-hash": "new-commit-hash" },
+    },
+  ],
+  deleted: [],
+};
+
+const fakeDafsaBinFile = do_get_file("data/fake_remote_dafsa.bin");
+const mockedFilePath = fakeDafsaBinFile.path;
+
+/**
+ * downloadCalled is used by mockDownload() and resetMockDownload()
+ * to keep track weather CLIENT.attachments.download is called or not
+ * downloadBackup will help restore CLIENT.attachments.download to original definition
+ * notifyUpdateBackup will help restore PublicSuffixList.notifyUpdate to original definition
+ */
+let downloadCalled = false;
+const downloadBackup = CLIENT.attachments.download;
+
+// returns a fake fileURI and sends a signal with filePath and no nsifile
+const mockDownload = () => {
+  downloadCalled = false;
+  CLIENT.attachments.download = async rec => {
+    downloadCalled = true;
+    return `file://${mockedFilePath}`; // Create a fake file URI
+  };
+};
+
+// resetMockDownload() must be run at the end of the test that uses mockDownload()
+const resetMockDownload = () => {
+  CLIENT.attachments.download = downloadBackup;
+};
+
+add_task(async () => {
+  info("File path sent when record is in DB.");
+
+  const collection = await CLIENT.openCollection();
+  await collection.clear(); // Make sure there's no record initially
+  await collection.create(
+    {
+      id: "tld-dafsa",
+      "commit-hash": "fake-commit-hash",
+      attachment: {},
+    },
+    { synced: true }
+  );
+
+  mockDownload();
+
+  const promiseSignal = TestUtils.topicObserved(SIGNAL);
+  await PublicSuffixList.init();
+  const observed = await promiseSignal;
+
+  Assert.equal(
+    observed[1],
+    mockedFilePath,
+    "File path sent when record is in DB."
+  );
+  await collection.clear(); // Clean up the mockDownloaded record
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("File path sent when record updated.");
+
+  mockDownload();
+
+  const promiseSignal = TestUtils.topicObserved(SIGNAL);
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_RECORDS });
+  const observed = await promiseSignal;
+
+  Assert.equal(
+    observed[1],
+    mockedFilePath,
+    "File path sent when record updated."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("Attachment downloaded when record created.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_CREATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    true,
+    "Attachment downloaded when record created."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("Attachment downloaded when record updated.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    true,
+    "Attachment downloaded when record updated."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("No download when more than one record is changed.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_AND_CREATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    false,
+    "No download when more than one record is changed."
+  );
+  resetMockDownload();
+});
new file mode 100644
--- /dev/null
+++ b/netwerk/dns/tests/unit/xpcshell.ini
@@ -0,0 +1,6 @@
+[DEFAULT]
+head = ../../../../services/common/tests/unit/head_global.js ../../../../services/common/tests/unit/head_helpers.js
+firefox-appdir = browser
+support-files = data/**
+
+[test_PublicSuffixList.js]