Bug 1331139 - Enable download protection for V4 lists. r=francois
authorThomas Nguyen <tnguyen@mozilla.com>
Mon, 27 Feb 2017 12:18:33 +0800
changeset 490224 2bd28e8cf65f33cb52cd2ab46172669433339a7e
parent 490223 9bec0be69329be127194c60ff6986efa549c35bc
child 490225 c2fbddc3469aeb1e47c33f37eeeea8352c01afd7
push id47030
push userbmo:ttromey@mozilla.com
push dateMon, 27 Feb 2017 20:32:32 +0000
reviewersfrancois
bugs1331139
milestone54.0a1
Bug 1331139 - Enable download protection for V4 lists. r=francois MozReview-Commit-ID: 8IHKGvXQjPt
modules/libpref/init/all.js
toolkit/components/downloads/ApplicationReputation.cpp
toolkit/components/downloads/ApplicationReputation.h
toolkit/components/downloads/moz.build
toolkit/components/downloads/test/gtest/TestLookupTable.cpp
toolkit/components/downloads/test/gtest/moz.build
toolkit/components/downloads/test/moz.build
toolkit/components/telemetry/Histograms.json
toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -5130,26 +5130,35 @@ pref("urlclassifier.malwareTable", "goog
 // In the official build, we are allowed to use google's private
 // phishing list "goog-phish-shavar". See Bug 1288840.
 pref("urlclassifier.phishTable", "goog-phish-shavar,test-phish-simple");
 #else
 pref("urlclassifier.phishTable", "googpub-phish-shavar,test-phish-simple");
 #endif
 
 // Tables for application reputation.
+#ifdef NIGHTLY_BUILD
+pref("urlclassifier.downloadBlockTable", "goog-badbinurl-shavar,goog-badbinurl-proto");
+#else
 pref("urlclassifier.downloadBlockTable", "goog-badbinurl-shavar");
+#endif
 
 #ifdef XP_WIN
  // Only download the whitelist on Windows, since the whitelist is
  // only useful for suppressing remote lookups for signed binaries which we can
  // only verify on Windows (Bug 974579). Other platforms always do remote lookups.
+#ifdef NIGHTLY_BUILD
+pref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256,goog-downloadwhite-proto");
+#else
 pref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256");
+#endif // NIGHTLY_BUILD
+
 #else
 pref("urlclassifier.downloadAllowTable", "");
-#endif
+#endif // XP_WIN
 
 pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,test-block-simple,test-flashallow-simple,testexcept-flashallow-simple,test-flash-simple,testexcept-flash-simple,test-flashsubdoc-simple,testexcept-flashsubdoc-simple,goog-downloadwhite-digest256,base-track-digest256,mozstd-trackwhite-digest256,content-track-digest256,mozplugin-block-digest256,mozplugin2-block-digest256");
 
 // The table and update/gethash URLs for Safebrowsing phishing and malware
 // checks.
 pref("urlclassifier.trackingTable", "test-track-simple,base-track-digest256");
 pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozstd-trackwhite-digest256");
 
@@ -5191,17 +5200,17 @@ pref("browser.safebrowsing.provider.goog
 pref("browser.safebrowsing.provider.google.gethashURL", "https://safebrowsing.google.com/safebrowsing/gethash?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2");
 pref("browser.safebrowsing.provider.google.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
 pref("browser.safebrowsing.provider.google.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
 pref("browser.safebrowsing.provider.google.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
 
 
 // Prefs for v4.
 pref("browser.safebrowsing.provider.google4.pver", "4");
-pref("browser.safebrowsing.provider.google4.lists", "goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto");
+pref("browser.safebrowsing.provider.google4.lists", "goog-badbinurl-proto,goog-downloadwhite-proto,goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto");
 pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%");
 // Leave it empty until we roll out v4 hash completion feature. See Bug 1323856.
 pref("browser.safebrowsing.provider.google4.gethashURL", "");
 pref("browser.safebrowsing.provider.google4.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
 pref("browser.safebrowsing.provider.google4.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
 pref("browser.safebrowsing.provider.google4.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
 
 pref("browser.safebrowsing.reportPhishURL", "https://%LOCALE%.phish-report.mozilla.com/?hl=%LOCALE%&url=");
--- a/toolkit/components/downloads/ApplicationReputation.cpp
+++ b/toolkit/components/downloads/ApplicationReputation.cpp
@@ -49,17 +49,19 @@
 #include "nsTArray.h"
 #include "nsThreadUtils.h"
 #include "nsXPCOMStrings.h"
 
 #include "nsIContentPolicy.h"
 #include "nsILoadInfo.h"
 #include "nsContentUtils.h"
 #include "nsWeakReference.h"
+#include "nsCharSeparatedTokenizer.h"
 
+using namespace mozilla::downloads;
 using mozilla::ArrayLength;
 using mozilla::BasePrincipal;
 using mozilla::OriginAttributes;
 using mozilla::Preferences;
 using mozilla::TimeStamp;
 using mozilla::Telemetry::Accumulate;
 using safe_browsing::ClientDownloadRequest;
 using safe_browsing::ClientDownloadRequest_CertificateChain;
@@ -82,16 +84,108 @@ using safe_browsing::ClientDownloadReque
 #define PREF_BLOCK_POTENTIALLY_UNWANTED "browser.safebrowsing.downloads.remote.block_potentially_unwanted"
 #define PREF_BLOCK_UNCOMMON             "browser.safebrowsing.downloads.remote.block_uncommon"
 
 // MOZ_LOG=ApplicationReputation:5
 mozilla::LazyLogModule ApplicationReputationService::prlog("ApplicationReputation");
 #define LOG(args) MOZ_LOG(ApplicationReputationService::prlog, mozilla::LogLevel::Debug, args)
 #define LOG_ENABLED() MOZ_LOG_TEST(ApplicationReputationService::prlog, mozilla::LogLevel::Debug)
 
+namespace mozilla {
+namespace downloads {
+
+enum class TelemetryMatchInfo : uint8_t
+{
+  eNoMatch   = 0x00,
+  eV2Match   = 0x01,
+  eV4Match   = 0x02,
+  eBothMatch = eV2Match | eV4Match,
+};
+
+MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(TelemetryMatchInfo)
+
+// Given a comma-separated list of tables which matched a URL, check to see if
+// at least one of these tables is present in the given pref.
+bool
+LookupTablesInPrefs(const nsACString& tables, const char* aPref)
+{
+  nsAutoCString prefList;
+  Preferences::GetCString(aPref, &prefList);
+  if (prefList.IsEmpty()) {
+    return false;
+  }
+
+  // Check if V2 and V4 are enabled in preference
+  // If V2 and V4 are both enabled, then we should do a telemetry record
+  // Both V2 and V4 begin with "goog" but V4 ends with "-proto"
+  nsCCharSeparatedTokenizer prefTokens(prefList, ',');
+  nsCString prefToken;
+  bool isV4Enabled = false;
+  bool isV2Enabled = false;
+
+  while (prefTokens.hasMoreTokens()) {
+    prefToken = prefTokens.nextToken();
+    if (StringBeginsWith(prefToken, NS_LITERAL_CSTRING("goog"))) {
+      if (StringEndsWith(prefToken, NS_LITERAL_CSTRING("-proto"))) {
+        isV4Enabled = true;
+      } else {
+        isV2Enabled = true;
+      }
+    }
+  }
+
+  bool shouldRecordTelemetry = isV2Enabled && isV4Enabled;
+  TelemetryMatchInfo telemetryInfo = TelemetryMatchInfo::eNoMatch;
+
+  // Parsed tables separated by "," into tokens then lookup each token
+  // in preference list
+  nsCCharSeparatedTokenizer tokens(tables, ',');
+  nsCString table;
+  bool found = false;
+
+  while (tokens.hasMoreTokens()) {
+    table = tokens.nextToken();
+    if (table.IsEmpty()) {
+      continue;
+    }
+
+    if (!FindInReadable(table, prefList)) {
+      continue;
+    }
+    found = true;
+
+    if (!shouldRecordTelemetry) {
+      return found;
+    }
+
+    // We are checking if the table found is V2 or V4 to record telemetry
+    // Both V2 and V4 begin with "goog" but V4 ends with "-proto"
+    if (StringBeginsWith(prefToken, NS_LITERAL_CSTRING("goog"))) {
+      if (StringEndsWith(prefToken, NS_LITERAL_CSTRING("-proto"))) {
+        telemetryInfo |= TelemetryMatchInfo::eV4Match;
+      } else {
+        telemetryInfo |= TelemetryMatchInfo::eV2Match;
+      }
+    }
+  }
+
+  // Record telemetry for matching allow list and block list
+  if (!strcmp(aPref, PREF_DOWNLOAD_BLOCK_TABLE)) {
+    Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_BLOCKLIST_MATCH,
+               static_cast<uint8_t>(telemetryInfo));
+  } else if (!strcmp(aPref, PREF_DOWNLOAD_ALLOW_TABLE)) {
+    Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_ALLOWLIST_MATCH,
+               static_cast<uint8_t>(telemetryInfo));
+  }
+
+  return found;
+}
+} // namespace downloads
+} // namespace mozilla
+
 class PendingDBLookup;
 
 // A single use class private to ApplicationReputationService encapsulating an
 // nsIApplicationReputationQuery and an nsIApplicationReputationCallback. Once
 // created by ApplicationReputationService, it is guaranteed to call mCallback.
 // This class is private to ApplicationReputationService.
 class PendingLookup final : public nsIStreamListener,
                             public nsITimerCallback,
@@ -343,29 +437,25 @@ PendingDBLookup::LookupSpecInternal(cons
 
 NS_IMETHODIMP
 PendingDBLookup::HandleEvent(const nsACString& tables)
 {
   // HandleEvent is guaranteed to call either:
   // 1) PendingLookup::OnComplete if the URL matches the blocklist, or
   // 2) PendingLookup::LookupNext if the URL does not match the blocklist.
   // Blocklisting trumps allowlisting.
-  nsAutoCString blockList;
-  Preferences::GetCString(PREF_DOWNLOAD_BLOCK_TABLE, &blockList);
-  if (!mAllowlistOnly && FindInReadable(blockList, tables)) {
+  if (!mAllowlistOnly && LookupTablesInPrefs(tables, PREF_DOWNLOAD_BLOCK_TABLE)) {
     mPendingLookup->mBlocklistCount++;
     Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, BLOCK_LIST);
     LOG(("Found principal %s on blocklist [this = %p]", mSpec.get(), this));
     return mPendingLookup->OnComplete(true, NS_OK,
       nsIApplicationReputationService::VERDICT_DANGEROUS);
   }
 
-  nsAutoCString allowList;
-  Preferences::GetCString(PREF_DOWNLOAD_ALLOW_TABLE, &allowList);
-  if (FindInReadable(allowList, tables)) {
+  if (LookupTablesInPrefs(tables, PREF_DOWNLOAD_ALLOW_TABLE)) {
     mPendingLookup->mAllowlistCount++;
     Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, ALLOW_LIST);
     LOG(("Found principal %s on allowlist [this = %p]", mSpec.get(), this));
     // Don't call onComplete, since blocklisting trumps allowlisting
   } else {
     LOG(("Didn't find principal %s on any list [this = %p]", mSpec.get(),
          this));
     Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, NO_LIST);
--- a/toolkit/components/downloads/ApplicationReputation.h
+++ b/toolkit/components/downloads/ApplicationReputation.h
@@ -15,16 +15,24 @@
 #include "nsCOMPtr.h"
 #include "nsString.h"
 #include "mozilla/Logging.h"
 
 class nsIRequest;
 class PendingDBLookup;
 class PendingLookup;
 
+namespace mozilla {
+namespace downloads {
+
+bool LookupTablesInPrefs(const nsACString& tables, const char* aPref);
+
+} // namespace downloads
+} // namespace mozilla
+
 class ApplicationReputationService final :
   public nsIApplicationReputationService {
 public:
   NS_DECL_ISUPPORTS
   NS_DECL_NSIAPPLICATIONREPUTATIONSERVICE
 
 public:
   static ApplicationReputationService* GetSingleton();
--- a/toolkit/components/downloads/moz.build
+++ b/toolkit/components/downloads/moz.build
@@ -14,17 +14,17 @@ with Files('chromium/*'):
     BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
 
 with Files('generate_csd.sh'):
     BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
 
 with Files('nsIApplicationReputation.idl'):
     BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
 
-XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
+TEST_DIRS += ['test']
 
 XPIDL_SOURCES += [
     'nsIApplicationReputation.idl',
     'nsIDownload.idl',
     'nsIDownloadManager.idl',
     'nsIDownloadManagerUI.idl',
     'nsIDownloadProgressListener.idl',
 ]
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/gtest/TestLookupTable.cpp
@@ -0,0 +1,47 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "ApplicationReputation.h"
+#include "mozilla/Preferences.h"
+#include "nsLiteralString.h"
+
+using namespace mozilla;
+using namespace mozilla::downloads;
+
+TEST(PendingLookup, LookupTablesInPrefs)
+{
+  EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", "goog-badbinurl-proto,goog-downloadwhite-proto,goog-badbinurl-shavar"));
+
+  bool result;
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,adafaf,,daf,goog-badbinurl-proto"), "gtest.test");
+  ASSERT_TRUE(result);
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,adafaf,,daf,goog-downloadwhite-proto"), "gtest.test");
+  ASSERT_TRUE(result);
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto"), "gtest.test");
+  ASSERT_FALSE(result);
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+  ASSERT_TRUE(result);
+
+  EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", "goog-badbinurl-proto"));
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-badbinurl-proto"), "gtest.test");
+  ASSERT_TRUE(result);
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+  ASSERT_TRUE(result);
+
+  // Empty prefrence
+  EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", ""));
+
+  result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+  ASSERT_FALSE(result);
+
+  Preferences::ClearUser("gtest.test");
+}
+
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/gtest/moz.build
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+LOCAL_INCLUDES += [
+    '../..',
+]
+
+UNIFIED_SOURCES += [
+    'TestLookupTable.cpp',
+]
+
+FINAL_LIBRARY = 'xul-gtest'
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/moz.build
@@ -0,0 +1,10 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
+
+if CONFIG['ENABLE_TESTS']:
+    DIRS += ['gtest']
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -116,16 +116,32 @@
   },
   "APPLICATION_REPUTATION_REMOTE_LOOKUP_TIMEOUT": {
     "alert_emails": ["safebrowsing-telemetry@mozilla.org"],
     "expires_in_version": "56",
     "kind": "boolean",
     "bug_numbers": [1172689],
     "description": "Recorded when application reputation remote lookup is performed, `true` is recorded if the lookup times out."
   },
+  "APPLICATION_REPUTATION_BLOCKLIST_MATCH": {
+    "alert_emails": ["safebrowsing-telemetry@mozilla.org"],
+    "expires_in_version": "60",
+    "kind": "enumerated",
+    "n_values": 4,
+    "bug_numbers": [1331139],
+    "description": "For each Application Reputation lookup against both the V2 and V4 Google lists, note which version of block list returned a match (0 = no match, 1 = match only V2, 2 = match only V4, 3 = match both V2 and V4)"
+  },
+  "APPLICATION_REPUTATION_ALLOWLIST_MATCH": {
+    "alert_emails": ["safebrowsing-telemetry@mozilla.org"],
+    "expires_in_version": "60",
+    "kind": "enumerated",
+    "n_values": 4,
+    "bug_numbers": [1331139],
+    "description": "For each Application Reputation lookup against both the V2 and V4 Google lists, note which version of the allow list returned a match (0 = no match, 1 = match only V2, 2 = match only V4, 3 = match both V2 and V4)"
+  },
   "AUDIOSTREAM_FIRST_OPEN_MS": {
     "expires_in_version": "50",
     "kind": "exponential",
     "high": 10000,
     "n_buckets": 50,
     "description": "The length of time (in milliseconds) for the first open of AudioStream."
   },
   "AUDIOSTREAM_LATER_OPEN_MS": {
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
@@ -226,16 +226,20 @@ static const struct {
   const char* mListName;
   uint32_t mThreatType;
 } THREAT_TYPE_CONV_TABLE[] = {
   { "goog-malware-proto",  MALWARE_THREAT},            // 1
   { "googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2
   { "goog-unwanted-proto", UNWANTED_SOFTWARE},         // 3
   { "goog-phish-proto", SOCIAL_ENGINEERING},           // 5
 
+  // For application reputation
+  { "goog-badbinurl-proto", MALICIOUS_BINARY},         // 7
+  { "goog-downloadwhite-proto", CSD_DOWNLOAD_WHITELIST},  // 9
+
   // For testing purpose.
   { "test-phish-proto",    SOCIAL_ENGINEERING_PUBLIC}, // 2
   { "test-unwanted-proto", UNWANTED_SOFTWARE}, // 3
 };
 
 NS_IMETHODIMP
 nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType,
                                                    nsACString& aListNames)