Bug 1511111 - Add nsIUrlClassifierSkipListService to integrate url-classifier with RemoteSettings and pref list updates. r=Ehsan,baku,leplatrem
authorJohann Hofmann <jhofmann@mozilla.com>
Fri, 08 Mar 2019 22:21:32 +0000
changeset 521202 5d96f8f2847e9992e7db9ac85b188ddfe1aa93ea
parent 521201 16951165b3632e4b4297d51abba927dd5aa1e201
child 521203 325d88b1233348f9984e6059011b787bd4129529
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersEhsan, baku, leplatrem
bugs1511111
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1511111 - Add nsIUrlClassifierSkipListService to integrate url-classifier with RemoteSettings and pref list updates. r=Ehsan,baku,leplatrem Differential Revision: https://phabricator.services.mozilla.com/D18597
browser/installer/package-manifest.in
mobile/android/installer/package-manifest.in
netwerk/url-classifier/UrlClassifierFeatureBase.cpp
netwerk/url-classifier/UrlClassifierFeatureBase.h
netwerk/url-classifier/UrlClassifierSkipListService.js
netwerk/url-classifier/UrlClassifierSkipListService.manifest
netwerk/url-classifier/moz.build
netwerk/url-classifier/nsIUrlClassifierFeature.idl
netwerk/url-classifier/nsIUrlClassifierSkipListService.idl
toolkit/components/url-classifier/tests/unit/test_SkipListService.js
toolkit/components/url-classifier/tests/unit/xpcshell.ini
--- a/browser/installer/package-manifest.in
+++ b/browser/installer/package-manifest.in
@@ -215,16 +215,20 @@
 @RESPATH@/browser/components/extensions-browser.manifest
 
 ; Modules
 @RESPATH@/browser/modules/*
 @RESPATH@/modules/*
 @RESPATH@/browser/actors/*
 @RESPATH@/actors/*
 
+; Safe Browsing
+@RESPATH@/components/UrlClassifierSkipListService.js
+@RESPATH@/components/UrlClassifierSkipListService.manifest
+
 ; ANGLE GLES-on-D3D rendering library
 #ifdef MOZ_ANGLE_RENDERER
 @BINPATH@/libEGL.dll
 @BINPATH@/libGLESv2.dll
 
 #ifdef MOZ_D3DCOMPILER_VISTA_DLL
 @BINPATH@/@MOZ_D3DCOMPILER_VISTA_DLL@
 #endif
--- a/mobile/android/installer/package-manifest.in
+++ b/mobile/android/installer/package-manifest.in
@@ -134,16 +134,20 @@
 @BINPATH@/components/TestInterfaceJS.manifest
 @BINPATH@/components/TestInterfaceJSMaplike.js
 #endif
 
 ; Modules
 @BINPATH@/modules/*
 @BINPATH@/actors/*
 
+; Safe Browsing
+@BINPATH@/components/UrlClassifierSkipListService.js
+@BINPATH@/components/UrlClassifierSkipListService.manifest
+
 ; [Browser Chrome Files]
 @BINPATH@/chrome/toolkit@JAREXT@
 @BINPATH@/chrome/toolkit.manifest
 
 ; [Extensions]
 @BINPATH@/components/extensions-toolkit.manifest
 @BINPATH@/components/extensions-mobile.manifest
 
--- a/netwerk/url-classifier/UrlClassifierFeatureBase.cpp
+++ b/netwerk/url-classifier/UrlClassifierFeatureBase.cpp
@@ -19,27 +19,22 @@ namespace {
 void OnPrefsChange(const char* aPrefName, nsTArray<nsCString>* aArray) {
   MOZ_ASSERT(aArray);
 
   nsAutoCString value;
   Preferences::GetCString(aPrefName, value);
   Classifier::SplitTables(value, *aArray);
 }
 
-void OnPrefSkipChange(const char* aPrefName, nsCString* aValue) {
-  MOZ_ASSERT(aValue);
-
-  Preferences::GetCString(aPrefName, *aValue);
-}
-
 }  // namespace
 
 NS_INTERFACE_MAP_BEGIN(UrlClassifierFeatureBase)
   NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIUrlClassifierFeature)
   NS_INTERFACE_MAP_ENTRY(nsIUrlClassifierFeature)
+  NS_INTERFACE_MAP_ENTRY(nsIUrlClassifierSkipListObserver)
 NS_INTERFACE_MAP_END
 
 NS_IMPL_ADDREF(UrlClassifierFeatureBase)
 NS_IMPL_RELEASE(UrlClassifierFeatureBase)
 
 UrlClassifierFeatureBase::UrlClassifierFeatureBase(
     const nsACString& aName, const nsACString& aPrefBlacklistTables,
     const nsACString& aPrefWhitelistTables,
@@ -73,41 +68,51 @@ void UrlClassifierFeatureBase::Initializ
     }
 
     if (!mPrefHosts[i].IsEmpty()) {
       Preferences::RegisterCallbackAndCall(OnPrefsChange, mPrefHosts[i],
                                            &mHosts[i]);
     }
   }
 
-  if (!mPrefSkipHosts.IsEmpty()) {
-    Preferences::RegisterCallbackAndCall(OnPrefSkipChange, mPrefSkipHosts,
-                                         &mSkipHosts);
+  nsCOMPtr<nsIUrlClassifierSkipListService> skipListService =
+      do_GetService("@mozilla.org/url-classifier/skip-list-service;1");
+  if (NS_WARN_IF(!skipListService)) {
+    return;
   }
+
+  skipListService->RegisterAndRunSkipListObserver(mName, mPrefSkipHosts, this);
 }
 
 void UrlClassifierFeatureBase::ShutdownPreferences() {
   for (uint32_t i = 0; i < 2; ++i) {
     if (!mPrefTables[i].IsEmpty()) {
       Preferences::UnregisterCallback(OnPrefsChange, mPrefTables[i],
                                       &mTables[i]);
     }
 
     if (!mPrefHosts[i].IsEmpty()) {
       Preferences::UnregisterCallback(OnPrefsChange, mPrefHosts[i], &mHosts[i]);
     }
   }
 
-  if (!mPrefSkipHosts.IsEmpty()) {
-    Preferences::UnregisterCallback(OnPrefSkipChange, mPrefSkipHosts,
-                                    &mSkipHosts);
+  nsCOMPtr<nsIUrlClassifierSkipListService> skipListService =
+      do_GetService("@mozilla.org/url-classifier/skip-list-service;1");
+  if (skipListService) {
+    skipListService->UnregisterSkipListObserver(mName, this);
   }
 }
 
 NS_IMETHODIMP
+UrlClassifierFeatureBase::OnSkipListUpdate(const nsACString& aList) {
+  mSkipHosts = aList;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
 UrlClassifierFeatureBase::GetName(nsACString& aName) {
   aName = mName;
   return NS_OK;
 }
 
 NS_IMETHODIMP
 UrlClassifierFeatureBase::GetTables(nsIUrlClassifierFeature::listType aListType,
                                     nsTArray<nsCString>& aTables) {
--- a/netwerk/url-classifier/UrlClassifierFeatureBase.h
+++ b/netwerk/url-classifier/UrlClassifierFeatureBase.h
@@ -3,24 +3,26 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef mozilla_net_UrlClassifierFeatureBase_h
 #define mozilla_net_UrlClassifierFeatureBase_h
 
 #include "nsIUrlClassifierFeature.h"
+#include "nsIUrlClassifierSkipListService.h"
 #include "nsTArray.h"
 #include "nsString.h"
 #include "mozilla/AntiTrackingCommon.h"
 
 namespace mozilla {
 namespace net {
 
-class UrlClassifierFeatureBase : public nsIUrlClassifierFeature {
+class UrlClassifierFeatureBase : public nsIUrlClassifierFeature,
+                                 public nsIUrlClassifierSkipListObserver {
  public:
   NS_DECL_ISUPPORTS
 
   NS_IMETHOD
   GetName(nsACString& aName) override;
 
   NS_IMETHOD
   GetTables(nsIUrlClassifierFeature::listType aListType,
@@ -33,16 +35,19 @@ class UrlClassifierFeatureBase : public 
   NS_IMETHOD
   HasHostInPreferences(const nsACString& aHost,
                        nsIUrlClassifierFeature::listType aListType,
                        nsACString& aPrefTableName, bool* aResult) override;
 
   NS_IMETHOD
   GetSkipHostList(nsACString& aList) override;
 
+  NS_IMETHOD
+  OnSkipListUpdate(const nsACString& aList) override;
+
  protected:
   UrlClassifierFeatureBase(const nsACString& aName,
                            const nsACString& aPrefBlacklistTables,
                            const nsACString& aPrefWhitelistTables,
                            const nsACString& aPrefBlacklistHosts,
                            const nsACString& aPrefWhitelistHosts,
                            const nsACString& aPrefBlacklistTableName,
                            const nsACString& aPrefWhitelistTableName,
new file mode 100644
--- /dev/null
+++ b/netwerk/url-classifier/UrlClassifierSkipListService.js
@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.UrlClassifierSkipListService = function() {};
+
+const {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+ChromeUtils.defineModuleGetter(this, "RemoteSettings",
+                               "resource://services-settings/remote-settings.js");
+
+const COLLECTION_NAME = "url-classifier-skip-urls";
+
+class Feature {
+  constructor(name, prefName) {
+    this.name = name;
+    this.prefName = prefName;
+    this.observers = new Set();
+    this.prefValue = null;
+    this.remoteEntries = null;
+
+    if (prefName) {
+      this.prefValue = Services.prefs.getStringPref(this.prefName, null);
+      Services.prefs.addObserver(prefName, this);
+    }
+
+    RemoteSettings(COLLECTION_NAME).on("sync", event => {
+      let { data: { current } } = event;
+      this.remoteEntries = current;
+      this.notifyObservers();
+    });
+  }
+
+  async addObserver(observer) {
+    // If the remote settings list hasn't been populated yet we have to make sure
+    // to do it before firing the first notification.
+    if (!this.remoteEntries) {
+      this.remoteEntries = await RemoteSettings(COLLECTION_NAME).get({ syncIfEmpty: false });
+    }
+
+    this.observers.add(observer);
+    this.notifyObservers(observer);
+  }
+
+  removeObserver(observer) {
+    this.observers.delete(observer);
+  }
+
+  observe(subject, topic, data) {
+    if (topic != "nsPref:changed" || data != this.prefName) {
+      Cu.reportError(`Unexpected event ${topic} with ${data}`);
+      return;
+    }
+
+    this.prefValue = Services.prefs.getStringPref(this.prefName, null);
+    this.notifyObservers();
+  }
+
+  notifyObservers(observer = null) {
+    let entries = [];
+    if (this.prefValue) {
+      entries = this.prefValue.split(",");
+    }
+
+    for (let entry of this.remoteEntries) {
+      if (entry.feature == this.name) {
+        entries.push(entry.pattern.toLowerCase());
+      }
+    }
+
+    let entriesAsString = entries.join(",");
+    if (observer) {
+      observer.onSkipListUpdate(entriesAsString);
+    } else {
+      for (let obs of this.observers) {
+        obs.onSkipListUpdate(entriesAsString);
+      }
+    }
+  }
+}
+
+UrlClassifierSkipListService.prototype = Object.freeze({
+  classID: Components.ID("{b9f4fd03-9d87-4bfd-9958-85a821750ddc}"),
+  QueryInterface: ChromeUtils.generateQI([Ci.nsIUrlClassifierSkipListService]),
+  _xpcom_factory: XPCOMUtils.generateSingletonFactory(UrlClassifierSkipListService),
+
+  features: {},
+
+  registerAndRunSkipListObserver(feature, prefName, observer) {
+    if (!this.features[feature]) {
+      this.features[feature] = new Feature(feature, prefName);
+    }
+    this.features[feature].addObserver(observer);
+  },
+
+  unregisterSkipListObserver(feature, observer) {
+    if (!this.features[feature]) {
+      return;
+    }
+    this.features[feature].removeObserver(observer);
+  },
+});
+
+this.NSGetFactory = XPCOMUtils.generateNSGetFactory([UrlClassifierSkipListService]);
new file mode 100644
--- /dev/null
+++ b/netwerk/url-classifier/UrlClassifierSkipListService.manifest
@@ -0,0 +1,2 @@
+component {b9f4fd03-9d87-4bfd-9958-85a821750ddc} UrlClassifierSkipListService.js
+contract @mozilla.org/url-classifier/skip-list-service;1 {b9f4fd03-9d87-4bfd-9958-85a821750ddc}
--- a/netwerk/url-classifier/moz.build
+++ b/netwerk/url-classifier/moz.build
@@ -5,20 +5,26 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files('**'):
     BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
 
 XPIDL_SOURCES += [
     'nsIURIClassifier.idl',
     'nsIUrlClassifierFeature.idl',
+    'nsIUrlClassifierSkipListService.idl',
 ]
 
 XPIDL_MODULE = 'url-classifier'
 
+EXTRA_COMPONENTS += [
+    'UrlClassifierSkipListService.js',
+    'UrlClassifierSkipListService.manifest',
+]
+
 DEFINES['GOOGLE_PROTOBUF_NO_RTTI'] = True
 DEFINES['GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER'] = True
 
 UNIFIED_SOURCES += [
     'AsyncUrlChannelClassifier.cpp',
     'nsChannelClassifier.cpp',
     'UrlClassifierCommon.cpp',
     'UrlClassifierFeatureBase.cpp',
--- a/netwerk/url-classifier/nsIUrlClassifierFeature.idl
+++ b/netwerk/url-classifier/nsIUrlClassifierFeature.idl
@@ -44,17 +44,17 @@ interface nsIUrlClassifierFeature : nsIS
    * Returns true if |aHost| is contained in the preference of |aListType| type.
    * |aPrefTableName| will be set to the table name to use.
    */
   [noscript] boolean hasHostInPreferences(in ACString aHost,
                                           in nsIUrlClassifierFeature_listType aListType,
                                           out ACString aPrefTableName);
 
   /**
-   * Returns true if this host has to be ignored also if blacklisted.
+   * Returns a comma-separated list of hosts to be ignored.
    */
   readonly attribute ACString skipHostList;
 
   /**
    * When this feature matches the channel, this method is executed to do
    * 'something' on the channel. For instance, a tracking-annotation feature
    * would mark the channel as tracker, a tracking-protection feature would
    * cancel the channel.
new file mode 100644
--- /dev/null
+++ b/netwerk/url-classifier/nsIUrlClassifierSkipListService.idl
@@ -0,0 +1,66 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+/**
+ * Observer for skip list updates.
+ */
+[scriptable, function, uuid(f7c918e5-94bf-4b6e-9758-ef7bdab6af7e)]
+interface nsIUrlClassifierSkipListObserver : nsISupports
+{
+  /**
+   * Called by nsIUrlClassifierSkipListService when the skip list
+   * for a designated feature changes and when the observer is first registered.
+   *
+   * @param aList
+   *        A comma-separated list of url patterns, intended to be parsed
+   *        by nsContentUtils::IsURIInList.
+   */
+  void onSkipListUpdate(in ACString aList);
+};
+
+/**
+ * A service that monitors updates to the skip list of url-classifier
+ * feature from sources such as a local pref and remote settings updates.
+ */
+[scriptable, uuid(75c3d1a3-e977-4079-9e27-b3b56bdb76ea)]
+interface nsIUrlClassifierSkipListService : nsISupports
+{
+  /**
+   * Register a new observer to skip list updates. When the observer is
+   * registered it is called immediately once. Afterwards it will be called
+   * whenever the specified pref changes or when remote settings for
+   * url-classifier features updates.
+   *
+   * @param aFeature
+   *        The feature for which to observe the skip list.
+   *
+   * @param aPrefName
+   *        (Optional) A pref name to monitor. The pref must be of string
+   *        type and contain a comma-separated list of URL patterns.
+   *
+   * @param aObserver
+   *        An nsIUrlClassifierSkipListObserver object or function that
+   *        will receive updates to the skip list as a comma-separated
+   *        string. Will be called immediately with the current skip
+   *        list value.
+   */
+  void registerAndRunSkipListObserver(in ACString aFeature,
+                                      in ACString aPrefName,
+                                      in nsIUrlClassifierSkipListObserver aObserver);
+
+  /**
+   * Unregister an observer.
+   *
+   * @param aFeature
+   *        The feature for which to stop observing.
+   *
+   * @param aObserver
+   *        The nsIUrlClassifierSkipListObserver object to unregister.
+   */
+  void unregisterSkipListObserver(in ACString aFeature,
+                                  in nsIUrlClassifierSkipListObserver aObserver);
+
+};
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/unit/test_SkipListService.js
@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* Unit tests for the nsIUrlClassifierSkipListService implementation. */
+
+const {RemoteSettings} = ChromeUtils.import("resource://services-settings/remote-settings.js");
+const {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const COLLECTION_NAME = "url-classifier-skip-urls";
+const FEATURE_NAME = "tracking-annotation-test";
+const FEATURE_PREF_NAME = "urlclassifier.tracking-annotation-test";
+
+XPCOMUtils.defineLazyGlobalGetters(this, ["EventTarget"]);
+
+do_get_profile();
+
+class UpdateEvent extends EventTarget { }
+function waitForEvent(element, eventName) {
+  return new Promise(function(resolve) {
+    element.addEventListener(eventName, e => resolve(e.detail), {once: true});
+  });
+}
+
+add_task(async function test_list_changes() {
+  let skipListService = Cc["@mozilla.org/url-classifier/skip-list-service;1"]
+    .getService(Ci.nsIUrlClassifierSkipListService);
+
+  // Make sure we have a pref initially, since the skip list service requires it.
+  Services.prefs.setStringPref(FEATURE_PREF_NAME, "");
+
+  let updateEvent = new UpdateEvent();
+  let obs = data => {
+    let event = new CustomEvent("update", { detail: data });
+    updateEvent.dispatchEvent(event);
+  };
+
+  let records = [{
+    id: "1",
+    last_modified: 100000000000000000001,
+    feature: FEATURE_NAME,
+    pattern: "example.com",
+  }];
+
+  // Add some initial data.
+  let collection = await RemoteSettings(COLLECTION_NAME).openCollection();
+  await collection.create(records[0], { synced: true });
+  await collection.db.saveLastModified(42);
+
+  let promise = waitForEvent(updateEvent, "update");
+
+  skipListService.registerAndRunSkipListObserver(FEATURE_NAME, FEATURE_PREF_NAME, obs);
+
+  let list = await promise;
+
+  Assert.equal(list, "example.com", "Has one item in the list");
+
+  records.push({
+    id: "2",
+    last_modified: 100000000000000000002,
+    feature: FEATURE_NAME,
+    pattern: "MOZILLA.ORG",
+  }, {
+    id: "3",
+    last_modified: 100000000000000000003,
+    feature: "some-other-feature",
+    pattern: "noinclude.com",
+  }, {
+    last_modified: 100000000000000000004,
+    feature: FEATURE_NAME,
+    pattern: "*.example.org",
+  });
+
+  promise = waitForEvent(updateEvent, "update");
+
+  await RemoteSettings(COLLECTION_NAME).emit("sync", { data: {current: records} });
+
+  list = await promise;
+
+  Assert.equal(list, "example.com,mozilla.org,*.example.org", "Has several items in the list");
+
+  promise = waitForEvent(updateEvent, "update");
+
+  Services.prefs.setStringPref(FEATURE_PREF_NAME, "test.com");
+
+  list = await promise;
+
+  Assert.equal(list, "test.com,example.com,mozilla.org,*.example.org", "Has several items in the list");
+
+  promise = waitForEvent(updateEvent, "update");
+
+  Services.prefs.setStringPref(FEATURE_PREF_NAME, "test.com,whatever.com,*.abc.com");
+
+  list = await promise;
+
+  Assert.equal(list, "test.com,whatever.com,*.abc.com,example.com,mozilla.org,*.example.org", "Has several items in the list");
+
+  skipListService.unregisterSkipListObserver(FEATURE_NAME, obs);
+
+  await collection.clear();
+});
--- a/toolkit/components/url-classifier/tests/unit/xpcshell.ini
+++ b/toolkit/components/url-classifier/tests/unit/xpcshell.ini
@@ -12,16 +12,17 @@ support-files =
 [test_hashcompleter.js]
 [test_hashcompleter_v4.js]
 # Bug 752243: Profile cleanup frequently fails
 #skip-if = os == "mac" || os == "linux"
 [test_partial.js]
 [test_prefixset.js]
 [test_threat_type_conversion.js]
 [test_provider_url.js]
+[test_SkipListService.js]
 [test_streamupdater.js]
 [test_digest256.js]
 [test_listmanager.js]
 [test_pref.js]
 [test_malwaretable_pref.js]
 [test_safebrowsing_protobuf.js]
 [test_platform_specific_threats.js]
 [test_features.js]