Bug 1141352 - add a pairwise allowlist to tracking protection. r=gcp
authorFrancois Marier <francois@mozilla.com>
Fri, 07 Aug 2015 13:08:22 -0700
changeset 288532 c58b7d331f5bc5b24bd1b0201e24c6adf6aed27e
parent 288531 803ba9e9abc703889ce73f0d4f16a2b411de2586
child 288533 2c5a611f298646326c73bdca44336fce92a59af0
push id5067
push userraliiev@mozilla.com
push dateMon, 21 Sep 2015 14:04:52 +0000
treeherdermozilla-beta@14221ffe5b2f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgcp
bugs1141352
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1141352 - add a pairwise allowlist to tracking protection. r=gcp
build/pgo/server-locations.txt
modules/libpref/init/all.js
netwerk/base/nsChannelClassifier.cpp
netwerk/base/nsChannelClassifier.h
netwerk/base/nsIURIClassifier.idl
netwerk/protocol/http/nsHttpChannel.cpp
toolkit/components/url-classifier/SafeBrowsing.jsm
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm
toolkit/components/url-classifier/tests/mochitest/chrome.ini
toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html
toolkit/components/url-classifier/tests/mochitest/good.js
toolkit/components/url-classifier/tests/mochitest/mochitest.ini
toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html
toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html
toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html
toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
--- a/build/pgo/server-locations.txt
+++ b/build/pgo/server-locations.txt
@@ -166,16 +166,18 @@ https://sub.sectest1.example.org:443
 
 #
 # Used while testing the url-classifier
 #
 http://malware.example.com:80
 http://tracking.example.com:80
 http://not-tracking.example.com:80
 http://tracking.example.org:80
+http://itisatracker.org:80
+http://trackertest.org:80
 
 https://malware.example.com:443
 https://tracking.example.com:443
 https://not-tracking.example.com:443
 https://tracking.example.org:443
 
 # Bug 483437, 484111
 https://www.bank1.com:443           privileged,cert=escapeattack1
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -4776,21 +4776,22 @@ pref("dom.inter-app-communication-api.en
 // Disable mapped array buffer by default.
 pref("dom.mapped_arraybuffer.enabled", false);
 
 // The tables used for Safebrowsing phishing and malware checks.
 pref("urlclassifier.malwareTable", "goog-malware-shavar,goog-unwanted-shavar,test-malware-simple,test-unwanted-simple");
 pref("urlclassifier.phishTable", "goog-phish-shavar,test-phish-simple");
 pref("urlclassifier.downloadBlockTable", "");
 pref("urlclassifier.downloadAllowTable", "");
-pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,goog-downloadwhite-digest256,mozpub-track-digest256");
+pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,goog-downloadwhite-digest256,mozpub-track-digest256,mozpub-trackwhite-digest256");
 
 // The table and update/gethash URLs for Safebrowsing phishing and malware
 // checks.
 pref("urlclassifier.trackingTable", "test-track-simple,mozpub-track-digest256");
+pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozpub-trackwhite-digest256");
 pref("browser.trackingprotection.updateURL", "https://tracking.services.mozilla.com/downloads?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2");
 pref("browser.trackingprotection.gethashURL", "https://tracking.services.mozilla.com/gethash?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2");
 
 // Turn off Spatial navigation by default.
 pref("snav.enabled", false);
 
 // Original caret implementation on collapsed selection.
 pref("touchcaret.enabled", false);
--- a/netwerk/base/nsChannelClassifier.cpp
+++ b/netwerk/base/nsChannelClassifier.cpp
@@ -507,22 +507,93 @@ nsChannelClassifier::SetBlockedTrackingC
                                   doc,
                                   nsContentUtils::eNECKO_PROPERTIES,
                                   "TrackingUriBlocked",
                                   params, ArrayLength(params));
 
   return NS_OK;
 }
 
+nsresult
+nsChannelClassifier::IsTrackerWhitelisted()
+{
+  nsresult rv;
+  nsCOMPtr<nsIURIClassifier> uriClassifier =
+    do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoCString tables;
+  Preferences::GetCString("urlclassifier.trackingWhitelistTable", &tables);
+
+  if (tables.IsEmpty()) {
+    LOG(("nsChannelClassifier[%p]:IsTrackerWhitelisted whitelist disabled",
+         this));
+    return NS_ERROR_TRACKING_URI;
+  }
+
+  nsCOMPtr<nsIHttpChannelInternal> chan = do_QueryInterface(mChannel, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsIURI> topWinURI;
+  rv = chan->GetTopWindowURI(getter_AddRefs(topWinURI));
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (!topWinURI) {
+    LOG(("nsChannelClassifier[%p]: No window URI", this));
+    return NS_ERROR_TRACKING_URI;
+  }
+
+  nsCOMPtr<nsIScriptSecurityManager> securityManager =
+    do_GetService(NS_SCRIPTSECURITYMANAGER_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsCOMPtr<nsIPrincipal> chanPrincipal;
+  rv = securityManager->GetChannelURIPrincipal(mChannel,
+                                               getter_AddRefs(chanPrincipal));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Craft a whitelist URL like "toplevel.page/?resource=third.party.domain"
+  nsAutoCString pageHostname, resourceDomain;
+  rv = topWinURI->GetHost(pageHostname);
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = chanPrincipal->GetBaseDomain(resourceDomain);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsAutoCString whitelistEntry = NS_LITERAL_CSTRING("http://") +
+    pageHostname + NS_LITERAL_CSTRING("/?resource=") + resourceDomain;
+  LOG(("nsChannelClassifier[%p]: Looking for %s in the whitelist",
+       this, whitelistEntry.get()));
+
+  nsCOMPtr<nsIURI> whitelistURI;
+  rv = NS_NewURI(getter_AddRefs(whitelistURI), whitelistEntry);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Check whether or not the tracker is in the entity whitelist
+  nsAutoCString results;
+  rv = uriClassifier->ClassifyLocalWithTables(whitelistURI, tables, results);
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (!results.IsEmpty()) {
+    return NS_OK; // found it on the whitelist, must not be blocked
+  }
+
+  LOG(("nsChannelClassifier[%p]: %s is not in the whitelist",
+       this, whitelistEntry.get()));
+  return NS_ERROR_TRACKING_URI;
+}
+
 NS_IMETHODIMP
 nsChannelClassifier::OnClassifyComplete(nsresult aErrorCode)
 {
     // Should only be called in the parent process.
     MOZ_ASSERT(XRE_IsParentProcess());
 
+    if (aErrorCode == NS_ERROR_TRACKING_URI &&
+        NS_SUCCEEDED(IsTrackerWhitelisted())) {
+      LOG(("nsChannelClassifier[%p]:OnClassifyComplete tracker found "
+           "in whitelist so we won't block it)", this));
+      aErrorCode = NS_OK;
+    }
+
     LOG(("nsChannelClassifier[%p]:OnClassifyComplete %d", this, aErrorCode));
     if (mSuspendedChannel) {
         MarkEntryClassified(aErrorCode);
 
         if (NS_FAILED(aErrorCode)) {
 #ifdef DEBUG
             nsCOMPtr<nsIURI> uri;
             mChannel->GetURI(getter_AddRefs(uri));
--- a/netwerk/base/nsChannelClassifier.h
+++ b/netwerk/base/nsChannelClassifier.h
@@ -36,16 +36,18 @@ private:
     ~nsChannelClassifier() {}
     // Caches good classifications for the channel principal.
     void MarkEntryClassified(nsresult status);
     bool HasBeenClassified(nsIChannel *aChannel);
     // Helper function so that we ensure we call ContinueBeginConnect once
     // Start is called. Returns NS_OK if and only if we will get a callback
     // from the classifier service.
     nsresult StartInternal();
+    // Helper function to check a tracking URI against the whitelist
+    nsresult IsTrackerWhitelisted();
 
 public:
     // If we are blocking tracking content, update the corresponding flag in
     // the respective docshell and call nsISecurityEventSink::onSecurityChange.
     static nsresult SetBlockedTrackingContent(nsIChannel *channel);
     static nsresult NotifyTrackingProtectionDisabled(nsIChannel *aChannel);
 };
 
--- a/netwerk/base/nsIURIClassifier.idl
+++ b/netwerk/base/nsIURIClassifier.idl
@@ -1,16 +1,17 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "nsISupports.idl"
 
+interface nsIChannel;
 interface nsIPrincipal;
-interface nsIChannel;
+interface nsIURI;
 
 /**
  * Callback function for nsIURIClassifier lookups.
  */
 [scriptable, function, uuid(8face46e-0c96-470f-af40-0037dcd797bd)]
 interface nsIURIClassifierCallback : nsISupports
 {
   /**
@@ -25,17 +26,17 @@ interface nsIURIClassifierCallback : nsI
    */
   void onClassifyComplete(in nsresult aErrorCode);
 };
 
 /**
  * The URI classifier service checks a URI against lists of phishing
  * and malware sites.
  */
-[scriptable, uuid(9168a330-7fba-40e8-9c47-4ce8f15a57fd)]
+[scriptable, uuid(596620cc-76e3-4133-9d90-360e59a794cf)]
 interface nsIURIClassifier : nsISupports
 {
   /**
    * Classify a Principal using its URI.
    *
    * @param aPrincipal
    *        The principal that should be checked by the URI classifier.
    * @param aTrackingProtectionEnabled
@@ -51,15 +52,14 @@ interface nsIURIClassifier : nsISupports
    *         <code>true</code> if classification will be performed.  The
    *         callback will be called.
    */
   boolean classify(in nsIPrincipal aPrincipal,
                    in boolean aTrackingProtectionEnabled,
                    in nsIURIClassifierCallback aCallback);
 
   /**
-   * Synchronously classify a Principal locally using its URI with a
-   * comma-separated string containing the given tables. This does not make
-   * network requests. The result is a comma-separated string of tables that match.
+   * Synchronously classify a URI with a comma-separated string
+   * containing the given tables. This does not make network requests.
+   * The result is a comma-separated string of tables that match.
    */
-  ACString classifyLocalWithTables(in nsIPrincipal aPrincipal,
-                                   in ACString aTables);
+  ACString classifyLocalWithTables(in nsIURI aURI, in ACString aTables);
 };
--- a/netwerk/protocol/http/nsHttpChannel.cpp
+++ b/netwerk/protocol/http/nsHttpChannel.cpp
@@ -5155,33 +5155,34 @@ nsHttpChannel::BeginConnect()
     // nsIHttpChannel.redirectTo API request
     if (mAPIRedirectToURI) {
         return AsyncCall(&nsHttpChannel::HandleAsyncAPIRedirect);
     }
     // Check to see if this principal exists on local blocklists.
     nsRefPtr<nsChannelClassifier> channelClassifier = new nsChannelClassifier();
     if (mLoadFlags & LOAD_CLASSIFY_URI) {
         nsCOMPtr<nsIURIClassifier> classifier = do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID);
-        if (classifier) {
-            bool tpEnabled = false;
-            channelClassifier->ShouldEnableTrackingProtection(this, &tpEnabled);
+        bool tpEnabled = false;
+        channelClassifier->ShouldEnableTrackingProtection(this, &tpEnabled);
+        if (classifier && tpEnabled) {
             // We skip speculative connections by setting mLocalBlocklist only
             // when tracking protection is enabled. Though we could do this for
             // both phishing and malware, it is not necessary for correctness,
             // since no network events will be received while the
             // nsChannelClassifier is in progress. See bug 1122691.
-            if (tpEnabled) {
-                nsCOMPtr<nsIPrincipal> principal = GetURIPrincipal();
+            nsCOMPtr<nsIURI> uri;
+            rv = GetURI(getter_AddRefs(uri));
+            if (NS_SUCCEEDED(rv) && uri) {
                 nsAutoCString tables;
                 Preferences::GetCString("urlclassifier.trackingTable", &tables);
                 nsAutoCString results;
-                rv = classifier->ClassifyLocalWithTables(principal, tables, results);
+                rv = classifier->ClassifyLocalWithTables(uri, tables, results);
                 if (NS_SUCCEEDED(rv) && !results.IsEmpty()) {
                     LOG(("nsHttpChannel::ClassifyLocalWithTables found "
-                         "principal on local tracking blocklist [this=%p]",
+                         "uri on local tracking blocklist [this=%p]",
                          this));
                     mLocalBlocklist = true;
                 } else {
                     LOG(("nsHttpChannel::ClassifyLocalWithTables no result "
                          "found [this=%p]", this));
                 }
             }
         }
--- a/toolkit/components/url-classifier/SafeBrowsing.jsm
+++ b/toolkit/components/url-classifier/SafeBrowsing.jsm
@@ -24,16 +24,17 @@ function getLists(prefName) {
 }
 
 // These may be a comma-separated lists of tables.
 const phishingLists = getLists("urlclassifier.phishTable");
 const malwareLists = getLists("urlclassifier.malwareTable");
 const downloadBlockLists = getLists("urlclassifier.downloadBlockTable");
 const downloadAllowLists = getLists("urlclassifier.downloadAllowTable");
 const trackingProtectionLists = getLists("urlclassifier.trackingTable");
+const trackingProtectionWhitelists = getLists("urlclassifier.trackingWhitelistTable");
 
 var debug = false;
 function log(...stuff) {
   if (!debug)
     return;
 
   var d = new Date();
   let msg = "SafeBrowsing: " + d.toTimeString() + ": " + stuff.join(" ");
@@ -68,16 +69,21 @@ this.SafeBrowsing = {
     for (let i = 0; i < downloadAllowLists.length; ++i) {
       listManager.registerTable(downloadAllowLists[i], this.updateURL, this.gethashURL);
     }
     for (let i = 0; i < trackingProtectionLists.length; ++i) {
       listManager.registerTable(trackingProtectionLists[i],
                                 this.trackingUpdateURL,
                                 this.trackingGethashURL);
     }
+    for (let i = 0; i < trackingProtectionWhitelists.length; ++i) {
+      listManager.registerTable(trackingProtectionWhitelists[i],
+                                this.trackingUpdateURL,
+                                this.trackingGethashURL);
+    }
     this.addMozEntries();
 
     this.controlUpdateChecking();
     this.initialized = true;
 
     log("init() finished");
   },
 
@@ -195,64 +201,71 @@ this.SafeBrowsing = {
         listManager.enableUpdate(downloadAllowLists[i]);
       } else {
         listManager.disableUpdate(downloadAllowLists[i]);
       }
     }
     for (let i = 0; i < trackingProtectionLists.length; ++i) {
       if (this.trackingEnabled) {
         listManager.enableUpdate(trackingProtectionLists[i]);
+        listManager.enableUpdate(trackingProtectionWhitelists[i]);
       } else {
         listManager.disableUpdate(trackingProtectionLists[i]);
+        listManager.disableUpdate(trackingProtectionWhitelists[i]);
       }
     }
     listManager.maybeToggleUpdateChecking();
   },
 
 
   addMozEntries: function() {
     // Add test entries to the DB.
     // XXX bug 779008 - this could be done by DB itself?
     const phishURL    = "itisatrap.org/firefox/its-a-trap.html";
     const malwareURL  = "itisatrap.org/firefox/its-an-attack.html";
     const unwantedURL = "itisatrap.org/firefox/unwanted.html";
-    const trackerURLs  = [
+    const trackerURLs = [
       "trackertest.org/",
       "itisatracker.org/",
     ];
+    const whitelistURL = "itisatrap.org/?resource=itisatracker.org";
 
     let update = "n:1000\ni:test-malware-simple\nad:1\n" +
                  "a:1:32:" + malwareURL.length + "\n" +
-                 malwareURL;
+                 malwareURL + "\n";
     update += "n:1000\ni:test-phish-simple\nad:1\n" +
               "a:1:32:" + phishURL.length + "\n" +
-              phishURL;
+              phishURL  + "\n";
     update += "n:1000\ni:test-unwanted-simple\nad:1\n" +
               "a:1:32:" + unwantedURL.length + "\n" +
-              unwantedURL;
+              unwantedURL + "\n";
+    update += "n:1000\ni:test-track-simple\n" +
+              "ad:" + trackerURLs.length + "\n";
     trackerURLs.forEach((trackerURL, i) => {
-      update += "n:1000\ni:test-track-simple\nad:1\n" +
-                "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" +
-                trackerURL;
+      update += "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" +
+                trackerURL + "\n";
     });
+    update += "n:1000\ni:test-trackwhite-simple\nad:1\n" +
+              "a:1:32:" + whitelistURL.length + "\n" +
+              whitelistURL;
     log("addMozEntries:", update);
 
     let db = Cc["@mozilla.org/url-classifier/dbservice;1"].
              getService(Ci.nsIUrlClassifierDBService);
 
     // nsIUrlClassifierUpdateObserver
     let dummyListener = {
       updateUrlRequested: function() { },
       streamFinished:     function() { },
       updateError:        function() { },
       updateSuccess:      function() { }
     };
 
     try {
-      let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple";
+      let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple";
       db.beginUpdate(dummyListener, tables, "");
       db.beginStream("", "");
       db.updateStream(update);
       db.finishStream();
       db.finishUpdate();
     } catch(ex) {
       // beginUpdate will throw harmlessly if there's an existing update in progress, ignore failures.
       log("addMozEntries failed!", ex);
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -73,16 +73,17 @@ PRLogModuleInfo *gUrlClassifierDbService
 
 #define GETHASH_NOISE_PREF      "urlclassifier.gethashnoise"
 #define GETHASH_NOISE_DEFAULT   4
 
 // Comma-separated lists
 #define MALWARE_TABLE_PREF      "urlclassifier.malwareTable"
 #define PHISH_TABLE_PREF        "urlclassifier.phishTable"
 #define TRACKING_TABLE_PREF     "urlclassifier.trackingTable"
+#define TRACKING_WHITELIST_TABLE_PREF "urlclassifier.trackingWhitelistTable"
 #define DOWNLOAD_BLOCK_TABLE_PREF "urlclassifier.downloadBlockTable"
 #define DOWNLOAD_ALLOW_TABLE_PREF "urlclassifier.downloadAllowTable"
 #define DISALLOW_COMPLETION_TABLE_PREF "urlclassifier.disallow_completions"
 
 #define CONFIRM_AGE_PREF        "urlclassifier.max-complete-age"
 #define CONFIRM_AGE_DEFAULT_SEC (45 * 60)
 
 class nsUrlClassifierDBServiceWorker;
@@ -1063,16 +1064,22 @@ nsUrlClassifierDBService::ReadTablesFrom
   }
 
   Preferences::GetCString(TRACKING_TABLE_PREF, &tables);
   if (!tables.IsEmpty()) {
     allTables.Append(',');
     allTables.Append(tables);
   }
 
+  Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
   Classifier::SplitTables(allTables, mGethashTables);
 
   Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, &tables);
   Classifier::SplitTables(tables, mDisallowCompletionsTables);
 
   return NS_OK;
 }
 
@@ -1110,16 +1117,17 @@ nsUrlClassifierDBService::Init()
   Preferences::AddStrongObserver(this, CHECK_PHISHING_PREF);
   Preferences::AddStrongObserver(this, CHECK_TRACKING_PREF);
   Preferences::AddStrongObserver(this, CHECK_TRACKING_PB_PREF);
   Preferences::AddStrongObserver(this, GETHASH_NOISE_PREF);
   Preferences::AddStrongObserver(this, CONFIRM_AGE_PREF);
   Preferences::AddStrongObserver(this, PHISH_TABLE_PREF);
   Preferences::AddStrongObserver(this, MALWARE_TABLE_PREF);
   Preferences::AddStrongObserver(this, TRACKING_TABLE_PREF);
+  Preferences::AddStrongObserver(this, TRACKING_WHITELIST_TABLE_PREF);
   Preferences::AddStrongObserver(this, DOWNLOAD_BLOCK_TABLE_PREF);
   Preferences::AddStrongObserver(this, DOWNLOAD_ALLOW_TABLE_PREF);
   Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF);
 
   // Force PSM loading on main thread
   nsresult rv;
   nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
@@ -1181,21 +1189,32 @@ nsUrlClassifierDBService::BuildTables(bo
     tables.Append(malware);
   }
   nsAutoCString phishing;
   Preferences::GetCString(PHISH_TABLE_PREF, &phishing);
   if (mCheckPhishing && !phishing.IsEmpty()) {
     tables.Append(',');
     tables.Append(phishing);
   }
-  nsAutoCString tracking;
-  Preferences::GetCString(TRACKING_TABLE_PREF, &tracking);
-  if (aTrackingProtectionEnabled && !tracking.IsEmpty()) {
-    tables.Append(',');
-    tables.Append(tracking);
+  if (aTrackingProtectionEnabled) {
+    nsAutoCString tracking, trackingWhitelist;
+    Preferences::GetCString(TRACKING_TABLE_PREF, &tracking);
+    if (!tracking.IsEmpty()) {
+      tables.Append(',');
+      tables.Append(tracking);
+    }
+    Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &trackingWhitelist);
+    if (!trackingWhitelist.IsEmpty()) {
+      tables.Append(',');
+      tables.Append(trackingWhitelist);
+    }
+  }
+
+  if (StringBeginsWith(tables, NS_LITERAL_CSTRING(","))) {
+    tables.Cut(0, 1);
   }
 }
 
 // nsChannelClassifier is the only consumer of this interface.
 NS_IMETHODIMP
 nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal,
                                    bool aTrackingProtectionEnabled,
                                    nsIURIClassifierCallback* c,
@@ -1224,35 +1243,30 @@ nsUrlClassifierDBService::Classify(nsIPr
     return NS_OK;
   }
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsUrlClassifierDBService::ClassifyLocalWithTables(nsIPrincipal *aPrincipal,
+nsUrlClassifierDBService::ClassifyLocalWithTables(nsIURI *aURI,
                                                   const nsACString & aTables,
                                                   nsACString & aTableResults)
 {
   MOZ_ASSERT(NS_IsMainThread(), "ClassifyLocalWithTables must be on main thread");
 
-  nsCOMPtr<nsIURI> uri;
-  nsresult rv = aPrincipal->GetURI(getter_AddRefs(uri));
-  NS_ENSURE_SUCCESS(rv, rv);
-  NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
-
-  uri = NS_GetInnermostURI(uri);
+  nsCOMPtr<nsIURI> uri = NS_GetInnermostURI(aURI);
   NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
 
   nsAutoCString key;
   // Canonicalize the url
   nsCOMPtr<nsIUrlClassifierUtils> utilsService =
     do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
-  rv = utilsService->GetKeyForURI(uri, key);
+  nsresult rv = utilsService->GetKeyForURI(uri, key);
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsAutoPtr<LookupResultArray> results(new LookupResultArray());
   if (!results) {
     return NS_ERROR_OUT_OF_MEMORY;
   }
 
   // In unittests, we may not have been initalized, so don't crash.
@@ -1508,16 +1522,17 @@ nsUrlClassifierDBService::Observe(nsISup
                NS_LITERAL_STRING(CHECK_TRACKING_PB_PREF).Equals(aData)) {
       mCheckTracking =
         Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) ||
         Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT);
     } else if (
       NS_LITERAL_STRING(PHISH_TABLE_PREF).Equals(aData) ||
       NS_LITERAL_STRING(MALWARE_TABLE_PREF).Equals(aData) ||
       NS_LITERAL_STRING(TRACKING_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(TRACKING_WHITELIST_TABLE_PREF).Equals(aData) ||
       NS_LITERAL_STRING(DOWNLOAD_BLOCK_TABLE_PREF).Equals(aData) ||
       NS_LITERAL_STRING(DOWNLOAD_ALLOW_TABLE_PREF).Equals(aData) ||
       NS_LITERAL_STRING(DISALLOW_COMPLETION_TABLE_PREF).Equals(aData)) {
       // Just read everything again.
       ReadTablesFromPrefs();
     } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) {
       gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF,
         CONFIRM_AGE_DEFAULT_SEC);
@@ -1547,16 +1562,17 @@ nsUrlClassifierDBService::Shutdown()
   if (prefs) {
     prefs->RemoveObserver(CHECK_MALWARE_PREF, this);
     prefs->RemoveObserver(CHECK_PHISHING_PREF, this);
     prefs->RemoveObserver(CHECK_TRACKING_PREF, this);
     prefs->RemoveObserver(CHECK_TRACKING_PB_PREF, this);
     prefs->RemoveObserver(PHISH_TABLE_PREF, this);
     prefs->RemoveObserver(MALWARE_TABLE_PREF, this);
     prefs->RemoveObserver(TRACKING_TABLE_PREF, this);
+    prefs->RemoveObserver(TRACKING_WHITELIST_TABLE_PREF, this);
     prefs->RemoveObserver(DOWNLOAD_BLOCK_TABLE_PREF, this);
     prefs->RemoveObserver(DOWNLOAD_ALLOW_TABLE_PREF, this);
     prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this);
     prefs->RemoveObserver(CONFIRM_AGE_PREF, this);
   }
 
   DebugOnly<nsresult> rv;
   // First close the db connection.
--- a/toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm
+++ b/toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm
@@ -1,45 +1,74 @@
 "use strict";
 
 this.EXPORTED_SYMBOLS = ["UrlClassifierTestUtils"];
 
 const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
 
+const TRACKING_TABLE_NAME = "mochitest-track-simple";
+const TRACKING_TABLE_PREF = "urlclassifier.trackingTable";
+const WHITELIST_TABLE_NAME = "mochitest-trackwhite-simple";
+const WHITELIST_TABLE_PREF = "urlclassifier.trackingWhitelistTable";
+
 Cu.import("resource://gre/modules/Services.jsm");
 
 this.UrlClassifierTestUtils = {
 
   addTestTrackers() {
-    const TABLE_PREF = "urlclassifier.trackingTable";
-    const TABLE_NAME = "test-track-simple";
+    // Add some URLs to the tracking databases
+    let trackingURL1 = "tracking.example.com/";
+    let trackingURL2 = "itisatracker.org/";
+    let trackingURL3 = "trackertest.org/";
+    let whitelistedURL = "itisatrap.org/?resource=itisatracker.org";
 
-    // Add some URLs to the tracking database (to be blocked)
-    let testData = "tracking.example.com/";
-    let testUpdate =
-          "n:1000\ni:test-track-simple\nad:1\n" +
-          "a:524:32:" + testData.length + "\n" +
-          testData;
+    let trackingUpdate =
+          "n:1000\ni:" + TRACKING_TABLE_NAME + "\nad:3\n" +
+          "a:1:32:" + trackingURL1.length + "\n" +
+          trackingURL1 + "\n" +
+          "a:2:32:" + trackingURL2.length + "\n" +
+          trackingURL2 + "\n" +
+          "a:3:32:" + trackingURL3.length + "\n" +
+          trackingURL3 + "\n";
+    let whitelistUpdate =
+          "n:1000\ni:" + WHITELIST_TABLE_NAME + "\nad:1\n" +
+          "a:1:32:" + whitelistedURL.length + "\n" +
+          whitelistedURL + "\n";
 
-    return this.useTestDatabase(TABLE_PREF, TABLE_NAME, testUpdate);
+    var tables = [
+      {
+        pref: TRACKING_TABLE_PREF,
+        name: TRACKING_TABLE_NAME,
+        update: trackingUpdate
+      },
+      {
+        pref: WHITELIST_TABLE_PREF,
+        name: WHITELIST_TABLE_NAME,
+        update: whitelistUpdate
+      }
+    ];
+
+    return this.useTestDatabase(tables);
   },
 
   cleanupTestTrackers() {
-    const TABLE_PREF = "urlclassifier.trackingTable";
-    Services.prefs.clearUserPref(TABLE_PREF);
+    Services.prefs.clearUserPref(TRACKING_TABLE_PREF);
+    Services.prefs.clearUserPref(WHITELIST_TABLE_PREF);
   },
 
   /**
    * Add some entries to a test tracking protection database, and resets
    * back to the default database after the test ends.
    *
    * @return {Promise}
    */
-  useTestDatabase(tablePref, tableName, update) {
-    Services.prefs.setCharPref(tablePref, tableName);
+  useTestDatabase(tables) {
+    for (var table of tables) {
+      Services.prefs.setCharPref(table.pref, table.name);
+    }
 
     return new Promise((resolve, reject) => {
       let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"].
                       getService(Ci.nsIUrlClassifierDBService);
       let listener = {
         QueryInterface: iid => {
           if (iid.equals(Ci.nsISupports) ||
               iid.equals(Ci.nsIUrlClassifierUpdateObserver))
@@ -52,16 +81,18 @@ this.UrlClassifierTestUtils = {
         updateError: errorCode => {
           reject("Couldn't update classifier.");
         },
         updateSuccess: requestedTimeout => {
           resolve();
         }
       };
 
-      dbService.beginUpdate(listener, tableName, "");
-      dbService.beginStream("", "");
-      dbService.updateStream(update);
-      dbService.finishStream();
-      dbService.finishUpdate();
+      for (var table of tables) {
+        dbService.beginUpdate(listener, table.name, "");
+        dbService.beginStream("", "");
+        dbService.updateStream(table.update);
+        dbService.finishStream();
+        dbService.finishUpdate();
+      }
     });
   },
 };
--- a/toolkit/components/url-classifier/tests/mochitest/chrome.ini
+++ b/toolkit/components/url-classifier/tests/mochitest/chrome.ini
@@ -9,8 +9,10 @@ support-files =
 [test_classified_annotations.html]
 tags = trackingprotection
 [test_allowlisted_annotations.html]
 tags = trackingprotection
 [test_privatebrowsing_trackingprotection.html]
 tags = trackingprotection
 [test_trackingprotection_bug1157081.html]
 tags = trackingprotection
+[test_trackingprotection_whitelist.html]
+tags = trackingprotection
--- a/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html
+++ b/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html
@@ -7,16 +7,18 @@
 
 <link id="badcss" rel="stylesheet" type="text/css" href="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link>
 
 </head>
 <body>
 
 <script id="badscript" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
 
+<script id="goodscript" data-touched="not sure" src="http://itisatracker.org/tests/toolkit/components/url-classifier/tests/mochitest/good.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
+
 <!-- The image cache can cache JS handlers, so make sure we use a different URL for raptor.jpg each time -->
 <img id="badimage" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg?pbmode=test" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"/>
 
 The following should not be hidden:
 <div id="styleCheck">STYLE TEST</div>
 
 </body>
 </html>
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/good.js
@@ -0,0 +1,1 @@
+scriptItem = "loaded whitelisted javascript!";
--- a/toolkit/components/url-classifier/tests/mochitest/mochitest.ini
+++ b/toolkit/components/url-classifier/tests/mochitest/mochitest.ini
@@ -1,16 +1,19 @@
 [DEFAULT]
 skip-if = buildapp == 'b2g' || e10s
 support-files =
+  classifiedAnnotatedPBFrame.html
   classifierFrame.html
   cleanWorker.js
+  good.js
   evil.css
   evil.js
   evilWorker.js
   import.css
   raptor.jpg
   track.html
   unwantedWorker.js
+  whitelistFrame.html
   workerFrame.html
 
 [test_classifier.html]
 [test_classifier_worker.html]
--- a/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html
+++ b/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html
@@ -20,17 +20,17 @@ var Cc = SpecialPowers.Cc;
 var Ci = SpecialPowers.Ci;
 
 var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor)
                     .getInterface(Ci.nsIWebNavigation)
                     .QueryInterface(Ci.nsIDocShellTreeItem)
                     .rootTreeItem
                     .QueryInterface(Ci.nsIInterfaceRequestor)
                     .getInterface(Ci.nsIDOMWindow);
-var contentPage = "chrome://mochitests/content/chrome/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html"
+var contentPage = "http://www.itisatrap.org/tests/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html";
 
 Components.utils.import("resource://gre/modules/Services.jsm");
 Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm");
 
 function whenDelayedStartupFinished(aWindow, aCallback) {
   Services.obs.addObserver(function observer(aSubject, aTopic) {
     if (aWindow == aSubject) {
       Services.obs.removeObserver(observer, aTopic);
@@ -66,16 +66,17 @@ var badids = [
   "badimage",
   "badcss"
 ];
 
 function checkLoads(aWindow, aBlocked) {
   var win = aWindow.content;
   is(win.document.getElementById("badscript").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking javascript");
   is(win.document.getElementById("badimage").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking images");
+  is(win.document.getElementById("goodscript").dataset.touched, "yes", "Should load whitelisted tracking javascript");
 
   var elt = win.document.getElementById("styleCheck");
   var style = win.document.defaultView.getComputedStyle(elt, "");
   isnot(style.visibility, aBlocked ? "hidden" : "", "Should not load tracking css");
 
   is(win.document.blockedTrackingNodeCount, aBlocked ? badids.length : 0, "Should identify all tracking elements");
 
   var blockedTrackingNodes = win.document.blockedTrackingNodes;
copy from toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html
copy to toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html
--- a/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html
+++ b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html
@@ -20,32 +20,33 @@ var Cc = SpecialPowers.Cc;
 var Ci = SpecialPowers.Ci;
 
 var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor)
                     .getInterface(Ci.nsIWebNavigation)
                     .QueryInterface(Ci.nsIDocShellTreeItem)
                     .rootTreeItem
                     .QueryInterface(Ci.nsIInterfaceRequestor)
                     .getInterface(Ci.nsIDOMWindow);
-var contentPage = "chrome://mochitests/content/chrome/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html"
+var contentPage1 = "http://www.itisatrap.org/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html";
+var contentPage2 = "http://example.com/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html";
 
 Components.utils.import("resource://gre/modules/Services.jsm");
 Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm");
 
 function whenDelayedStartupFinished(aWindow, aCallback) {
   Services.obs.addObserver(function observer(aSubject, aTopic) {
     if (aWindow == aSubject) {
       Services.obs.removeObserver(observer, aTopic);
       setTimeout(aCallback, 0);
     }
   }, "browser-delayed-startup-finished", false);
 }
 
-function testOnWindow(aPrivate, aCallback) {
-  var win = mainWindow.OpenBrowserWindow({private: aPrivate});
+function testOnWindow(contentPage, aCallback) {
+  var win = mainWindow.OpenBrowserWindow();
   win.addEventListener("load", function onLoad() {
     win.removeEventListener("load", onLoad, false);
     whenDelayedStartupFinished(win, function() {
       win.addEventListener("DOMContentLoaded", function onInnerLoad() {
         if (win.content.location.href != contentPage) {
           win.gBrowser.loadURI(contentPage);
           return;
         }
@@ -56,32 +57,30 @@ function testOnWindow(aPrivate, aCallbac
           SimpleTest.executeSoon(function() { aCallback(win); });
         }, false, true);
       }, true);
       SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); });
     });
   }, true);
 }
 
-var badids = [
+var alwaysbadids = [
   "badscript",
-  "badimage",
-  "badcss"
 ];
 
-function checkLoads(aWindow, aBlocked) {
+function checkLoads(aWindow, aWhitelisted) {
   var win = aWindow.content;
-  is(win.document.getElementById("badscript").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking javascript");
-  is(win.document.getElementById("badimage").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking images");
+  is(win.document.getElementById("badscript").dataset.touched, "no", "Should not load tracking javascript");
+  is(win.document.getElementById("goodscript").dataset.touched, aWhitelisted ? "yes" : "no", "Should load whitelisted tracking javascript");
 
-  var elt = win.document.getElementById("styleCheck");
-  var style = win.document.defaultView.getComputedStyle(elt, "");
-  isnot(style.visibility, aBlocked ? "hidden" : "", "Should not load tracking css");
-
-  is(win.document.blockedTrackingNodeCount, aBlocked ? badids.length : 0, "Should identify all tracking elements");
+  var badids = alwaysbadids.slice();
+  if (!aWhitelisted) {
+    badids.push("goodscript");
+  }
+  is(win.document.blockedTrackingNodeCount, badids.length, "Should identify all tracking elements");
 
   var blockedTrackingNodes = win.document.blockedTrackingNodes;
 
   // Make sure that every node in blockedTrackingNodes exists in the tree
   // (that may not always be the case but do not expect any nodes to disappear
   // from the tree here)
   var allNodeMatch = true;
   for (var i = 0; i < blockedTrackingNodes.length; i++) {
@@ -103,43 +102,41 @@ function checkLoads(aWindow, aBlocked) {
     var nodeMatch = false;
     for (var i = 0; i < blockedTrackingNodes.length && !nodeMatch; i++) {
       nodeMatch = nodeMatch ||
         (blockedTrackingNodes[i] == win.document.getElementById(badids[j]));
     }
 
     allNodeMatch = allNodeMatch && nodeMatch;
   }
-  is(allNodeMatch, aBlocked, "All tracking nodes are expected to be annotated as such");
+  is(allNodeMatch, true, "All tracking nodes are expected to be annotated as such");
 }
 
 SpecialPowers.pushPrefEnv(
-  {"set" : [["urlclassifier.trackingTable", "test-track-simple"],
-            ["privacy.trackingprotection.enabled", false],
-            ["privacy.trackingprotection.pbmode.enabled", true],
+  {"set" : [["privacy.trackingprotection.enabled", true],
             ["channelclassifier.allowlist_example", true]]},
   test);
 
 function test() {
   SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers);
   UrlClassifierTestUtils.addTestTrackers().then(() => {
-    // Normal mode, with the pref (trackers should be loaded)
-    testOnWindow(false, function(aWindow) {
-      checkLoads(aWindow, false);
+    // Load the test from a URL on the whitelist
+    testOnWindow(contentPage1, function(aWindow) {
+      checkLoads(aWindow, true);
       aWindow.close();
 
-      // Private Browsing, with the pref (trackers should be blocked)
-      testOnWindow(true, function(aWindow) {
-        checkLoads(aWindow, true);
+      // Load the test from a URL that's NOT on the whitelist
+      testOnWindow(contentPage2, function(aWindow) {
+        checkLoads(aWindow, false);
         aWindow.close();
 
-        // Private Browsing, without the pref (trackers should be loaded)
-        SpecialPowers.setBoolPref("privacy.trackingprotection.pbmode.enabled", false);
-        testOnWindow(true, function(aWindow) {
-          checkLoads(aWindow, false);
+        // Load the test from a URL on the whitelist but without the whitelist
+        SpecialPowers.setCharPref("urlclassifier.trackingWhitelistTable", "");
+        testOnWindow(contentPage1, function(aWindow) {
+        checkLoads(aWindow, false);
           aWindow.close();
           SimpleTest.finish();
         });
       });
     });
   });
 }
 
copy from toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html
copy to toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html
--- a/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html
+++ b/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html
@@ -1,22 +1,15 @@
 <!DOCTYPE HTML>
 <!-- Any copyright is dedicated to the Public Domain.
      http://creativecommons.org/publicdomain/zero/1.0/ -->
 <html>
 <head>
 <title></title>
-
-<link id="badcss" rel="stylesheet" type="text/css" href="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link>
-
 </head>
 <body>
 
-<script id="badscript" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
+<script id="badscript" data-touched="not sure" src="http://trackertest.org/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
 
-<!-- The image cache can cache JS handlers, so make sure we use a different URL for raptor.jpg each time -->
-<img id="badimage" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg?pbmode=test" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"/>
-
-The following should not be hidden:
-<div id="styleCheck">STYLE TEST</div>
+<script id="goodscript" data-touched="not sure" src="http://itisatracker.org/tests/toolkit/components/url-classifier/tests/mochitest/good.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
 
 </body>
 </html>
--- a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
+++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
@@ -50,29 +50,32 @@ function delFile(name) {
 
 function cleanUp() {
   delFile("urlclassifier3.sqlite");
   delFile("safebrowsing/classifier.hashkey");
   delFile("safebrowsing/test-phish-simple.sbstore");
   delFile("safebrowsing/test-malware-simple.sbstore");
   delFile("safebrowsing/test-unwanted-simple.sbstore");
   delFile("safebrowsing/test-track-simple.sbstore");
+  delFile("safebrowsing/test-trackwhite-simple.sbstore");
   delFile("safebrowsing/test-phish-simple.cache");
   delFile("safebrowsing/test-malware-simple.cache");
   delFile("safebrowsing/test-unwanted-simple.cache");
   delFile("safebrowsing/test-track-simple.cache");
+  delFile("safebrowsing/test-trackwhite-simple.cache");
   delFile("safebrowsing/test-phish-simple.pset");
   delFile("safebrowsing/test-malware-simple.pset");
   delFile("safebrowsing/test-unwanted-simple.pset");
   delFile("safebrowsing/test-track-simple.pset");
+  delFile("safebrowsing/test-trackwhite-simple.pset");
   delFile("testLarge.pset");
   delFile("testNoDelta.pset");
 }
 
-var allTables = "test-phish-simple,test-malware-simple,test-unwanted-simple,test-track-simple";
+var allTables = "test-phish-simple,test-malware-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple";
 
 var dbservice = Cc["@mozilla.org/url-classifier/dbservice;1"].getService(Ci.nsIUrlClassifierDBService);
 var streamUpdater = Cc["@mozilla.org/url-classifier/streamupdater;1"]
                     .getService(Ci.nsIUrlClassifierStreamUpdater);
 
 
 /*
  * Builds an update from an object that looks like: