Bug 1305486 - Enable v4 tables on nightly build. r=francois
authorHenry Chang <hchang@mozilla.com>
Sun, 20 Nov 2016 15:10:19 +0800
changeset 374121 ca2fd69cb777b9eae0950f997aeb188ebd9c2fd8
parent 374120 b351c04d0dc84965f8f8f3212272d21e53417599
child 374122 8091269b852c0452661da4e34b203d004b3f1781
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfrancois
bugs1305486
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1305486 - Enable v4 tables on nightly build. r=francois MozReview-Commit-ID: 4iGduuYqMQs
addon-sdk/source/python-lib/cuddlefish/prefs.py
addon-sdk/source/test/preferences/firefox.json
addon-sdk/source/test/preferences/no-connections.json
browser/app/profile/firefox.js
testing/firefox-ui/tests/functional/security/test_safe_browsing_initial_download.py
testing/profiles/prefs_general.js
testing/talos/talos/config.py
--- a/addon-sdk/source/python-lib/cuddlefish/prefs.py
+++ b/addon-sdk/source/python-lib/cuddlefish/prefs.py
@@ -53,16 +53,18 @@ DEFAULT_NO_CONNECTIONS_PREFS = {
     'media.gmp-manager.updateEnabled': False,
     'browser.aboutHomeSnippets.updateUrl': 'https://localhost/snippet-dummy',
     'browser.newtab.url' : 'about:blank',
     'browser.search.update': False,
     'browser.search.suggest.enabled' : False,
     'browser.safebrowsing.phishing.enabled' : False,
     'browser.safebrowsing.provider.google.updateURL': 'http://localhost/safebrowsing-dummy/update',
     'browser.safebrowsing.provider.google.gethashURL': 'http://localhost/safebrowsing-dummy/gethash',
+    'browser.safebrowsing.provider.google4.updateURL': 'http://localhost/safebrowsing4-dummy/update',
+    'browser.safebrowsing.provider.google4.gethashURL': 'http://localhost/safebrowsing4-dummy/gethash',
     'browser.safebrowsing.malware.reportURL': 'http://localhost/safebrowsing-dummy/malwarereport',
     'browser.selfsupport.url': 'https://localhost/selfsupport-dummy',
     'browser.safebrowsing.provider.mozilla.gethashURL': 'http://localhost/safebrowsing-dummy/gethash',
     'browser.safebrowsing.provider.mozilla.updateURL': 'http://localhost/safebrowsing-dummy/update',
 
     # Disable app update
     'app.update.enabled' : False,
     'app.update.staging.enabled': False,
@@ -114,16 +116,18 @@ DEFAULT_FIREFOX_PREFS = {
 
     # From:
     # http://hg.mozilla.org/mozilla-central/file/1dd81c324ac7/build/automation.py.in#l388
     # Make url-classifier updates so rare that they won't affect tests.
     'urlclassifier.updateinterval' : 172800,
     # Point the url-classifier to a nonexistent local URL for fast failures.
     'browser.safebrowsing.provider.google.gethashURL' : 'http://localhost/safebrowsing-dummy/gethash',
     'browser.safebrowsing.provider.google.updateURL' : 'http://localhost/safebrowsing-dummy/update',
+    'browser.safebrowsing.provider.google4.gethashURL' : 'http://localhost/safebrowsing4-dummy/gethash',
+    'browser.safebrowsing.provider.google4.updateURL' : 'http://localhost/safebrowsing4-dummy/update',
     'browser.safebrowsing.provider.mozilla.gethashURL': 'http://localhost/safebrowsing-dummy/gethash',
     'browser.safebrowsing.provider.mozilla.updateURL': 'http://localhost/safebrowsing-dummy/update',
 }
 
 # When launching a temporary new Thunderbird profile, use these preferences.
 # Note that these were taken from:
 # http://dxr.mozilla.org/comm-central/source/mail/test/mozmill/runtest.py
 DEFAULT_THUNDERBIRD_PREFS = {
--- a/addon-sdk/source/test/preferences/firefox.json
+++ b/addon-sdk/source/test/preferences/firefox.json
@@ -1,11 +1,13 @@
 {
   "browser.startup.homepage": "about:blank",
   "startup.homepage_welcome_url": "about:blank",
   "devtools.browsertoolbox.panel": "jsdebugger",
   "devtools.chrome.enabled": true,
   "urlclassifier.updateinterval": 172800,
   "browser.safebrowsing.provider.google.gethashURL": "http://localhost/safebrowsing-dummy/gethash",
   "browser.safebrowsing.provider.google.updateURL": "http://localhost/safebrowsing-dummy/update",
+  "browser.safebrowsing.provider.google4.gethashURL": "http://localhost/safebrowsing4-dummy/gethash",
+  "browser.safebrowsing.provider.google4.updateURL": "http://localhost/safebrowsing4-dummy/update",
   "browser.safebrowsing.provider.mozilla.gethashURL": "http://localhost/safebrowsing-dummy/gethash",
   "browser.safebrowsing.provider.mozilla.updateURL": "http://localhost/safebrowsing-dummy/update"
 }
--- a/addon-sdk/source/test/preferences/no-connections.json
+++ b/addon-sdk/source/test/preferences/no-connections.json
@@ -14,16 +14,19 @@
   "browser.aboutHomeSnippets.updateUrl": "https://localhost/snippet-dummy",
   "browser.newtab.url": "about:blank",
   "browser.search.update": false,
   "browser.search.suggest.enabled": false,
   "browser.safebrowsing.phishing.enabled": false,
   "browser.safebrowsing.provider.google.updateURL": "http://localhost/safebrowsing-dummy/update",
   "browser.safebrowsing.provider.google.gethashURL": "http://localhost/safebrowsing-dummy/gethash",
   "browser.safebrowsing.provider.google.reportURL": "http://localhost/safebrowsing-dummy/malwarereport",
+  "browser.safebrowsing.provider.google4.updateURL": "http://localhost/safebrowsing4-dummy/update",
+  "browser.safebrowsing.provider.google4.gethashURL": "http://localhost/safebrowsing4-dummy/gethash",
+  "browser.safebrowsing.provider.google4.reportURL": "http://localhost/safebrowsing4-dummy/malwarereport",
   "browser.selfsupport.url": "https://localhost/selfsupport-dummy",
   "browser.safebrowsing.provider.mozilla.gethashURL": "http://localhost/safebrowsing-dummy/gethash",
   "browser.safebrowsing.provider.mozilla.updateURL": "http://localhost/safebrowsing-dummy/update",
   "browser.newtabpage.directory.source": "data:application/json,{'jetpack':1}",
   "browser.newtabpage.directory.ping": "",
   "extensions.update.url": "http://localhost/extensions-dummy/updateURL",
   "extensions.update.background.url": "http://localhost/extensions-dummy/updateBackgroundURL",
   "extensions.blocklist.url": "http://localhost/extensions-dummy/blocklistURL",
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1560,8 +1560,14 @@ pref("browser.crashReports.unsubmittedCh
 #ifdef NIGHTLY_BUILD
 // Enable the (fairly costly) client/server validation on nightly only. The other prefs
 // controlling validation are located in /services/sync/services-sync.js
 pref("services.sync.validation.enabled", true);
 #endif
 
 // Preferences for the form autofill system extension
 pref("browser.formautofill.experimental", false);
+
+// Enable safebrowsing v4 tables (suffixed by "-proto") update.
+#ifdef NIGHTLY_BUILD
+pref("urlclassifier.malwareTable", "goog-malware-shavar,goog-unwanted-shavar,goog-malware-proto,goog-unwanted-proto,test-malware-simple,test-unwanted-simple");
+pref("urlclassifier.phishTable", "goog-phish-shavar,goog-phish-proto,test-phish-simple");
+#endif
\ No newline at end of file
--- a/testing/firefox-ui/tests/functional/security/test_safe_browsing_initial_download.py
+++ b/testing/firefox-ui/tests/functional/security/test_safe_browsing_initial_download.py
@@ -6,67 +6,86 @@ import os
 
 from firefox_puppeteer import PuppeteerMixin
 from marionette_driver import Wait
 from marionette_harness import MarionetteTestCase
 
 
 class TestSafeBrowsingInitialDownload(PuppeteerMixin, MarionetteTestCase):
 
-    file_extensions = [
+    v2_file_extensions = [
         'pset',
         'sbstore',
     ]
 
+    v4_file_extensions = [
+        'pset',
+        'metadata',
+    ]
+
     prefs_download_lists = [
         'urlclassifier.blockedTable',
         'urlclassifier.downloadAllowTable',
         'urlclassifier.downloadBlockTable',
         'urlclassifier.malwareTable',
         'urlclassifier.phishTable',
         'urlclassifier.trackingTable',
         'urlclassifier.trackingWhitelistTable',
     ]
 
     prefs_provider_update_time = {
         # Force an immediate download of the safebrowsing files
+        # Bug 1330253 - Leave the next line disabled until we have google API key
+        #               on the CI machines.
+        # 'browser.safebrowsing.provider.google4.nextupdatetime': 1,
         'browser.safebrowsing.provider.google.nextupdatetime': 1,
         'browser.safebrowsing.provider.mozilla.nextupdatetime': 1,
     }
 
     prefs_safebrowsing = {
         'browser.safebrowsing.debug': True,
         'browser.safebrowsing.blockedURIs.enabled': True,
         'browser.safebrowsing.downloads.enabled': True,
         'browser.safebrowsing.phishing.enabled': True,
         'browser.safebrowsing.malware.enabled': True,
         'privacy.trackingprotection.enabled': True,
         'privacy.trackingprotection.pbmode.enabled': True,
     }
 
-    def get_safebrowsing_files(self):
+    def get_safebrowsing_files(self, is_v4):
         files = []
+
+        if is_v4:
+            my_file_extensions = self.v4_file_extensions
+        else:  # v2
+            # safebrowsing dir should have a 'google4' directory where
+            # v4 databases exist.
+            files.append('google4')
+            my_file_extensions = self.v2_file_extensions
+
         for pref_name in self.prefs_download_lists:
             base_names = self.marionette.get_pref(pref_name).split(',')
-            for ext in self.file_extensions:
-                files.extend(['{file}.{ext}'.format(file=f, ext=ext) for f in base_names if f])
+            for ext in my_file_extensions:
+                files.extend(['{file}.{ext}'.format(file=f, ext=ext)
+                              for f in base_names if f and f.endswith('-proto') == is_v4])
 
         return set(sorted(files))
 
     def setUp(self):
         super(TestSafeBrowsingInitialDownload, self).setUp()
 
         # Force the preferences for the new profile
         enforce_prefs = self.prefs_safebrowsing
         enforce_prefs.update(self.prefs_provider_update_time)
         self.marionette.enforce_gecko_prefs(enforce_prefs)
 
         self.safebrowsing_path = os.path.join(self.marionette.instance.profile.profile,
                                               'safebrowsing')
-        self.safebrowsing_files = self.get_safebrowsing_files()
+        self.safebrowsing_v2_files = self.get_safebrowsing_files(False)
+        self.safebrowsing_v4_files = self.get_safebrowsing_files(True)
 
     def tearDown(self):
         try:
             # Restart with a fresh profile
             self.restart(clean=True)
         finally:
             super(TestSafeBrowsingInitialDownload, self).tearDown()
 
@@ -74,9 +93,14 @@ class TestSafeBrowsingInitialDownload(Pu
         def check_downloaded(_):
             return reduce(lambda state, pref: state and int(self.marionette.get_pref(pref)) != 1,
                           self.prefs_provider_update_time.keys(), True)
 
         try:
             Wait(self.marionette, timeout=60).until(
                 check_downloaded, message='Not all safebrowsing files have been downloaded')
         finally:
-            self.assertSetEqual(self.safebrowsing_files, set(os.listdir(self.safebrowsing_path)))
+            self.assertSetEqual(self.safebrowsing_v2_files,
+                                set(os.listdir(self.safebrowsing_path)))
+            # Bug 1330253 - Leave the next test disabled until we have google api key
+            #               on the CI machines.
+            # self.assertSetEqual(self.safebrowsing_v4_files,
+            #                     set(os.listdir(os.path.join(self.safebrowsing_path, 'google4'))))
--- a/testing/profiles/prefs_general.js
+++ b/testing/profiles/prefs_general.js
@@ -94,16 +94,18 @@ user_pref("geo.wifi.logging.enabled", tr
 user_pref("dom.push.connection.enabled", false);
 
 // Make url-classifier updates so rare that they won't affect tests
 user_pref("urlclassifier.updateinterval", 172800);
 // Point the url-classifier to the local testing server for fast failures
 user_pref("browser.safebrowsing.downloads.remote.url", "http://%(server)s/safebrowsing-dummy/update");
 user_pref("browser.safebrowsing.provider.google.gethashURL", "http://%(server)s/safebrowsing-dummy/gethash");
 user_pref("browser.safebrowsing.provider.google.updateURL", "http://%(server)s/safebrowsing-dummy/update");
+user_pref("browser.safebrowsing.provider.google4.gethashURL", "http://%(server)s/safebrowsing4-dummy/gethash");
+user_pref("browser.safebrowsing.provider.google4.updateURL", "http://%(server)s/safebrowsing4-dummy/update");
 user_pref("browser.safebrowsing.provider.mozilla.gethashURL", "http://%(server)s/safebrowsing-dummy/gethash");
 user_pref("browser.safebrowsing.provider.mozilla.updateURL", "http://%(server)s/safebrowsing-dummy/update");
 user_pref("privacy.trackingprotection.introURL", "http://%(server)s/trackingprotection/tour");
 // Point update checks to the local testing server for fast failures
 user_pref("extensions.update.url", "http://%(server)s/extensions-dummy/updateURL");
 user_pref("extensions.update.background.url", "http://%(server)s/extensions-dummy/updateBackgroundURL");
 user_pref("extensions.blocklist.detailsURL", "http://%(server)s/extensions-dummy/blocklistDetailsURL");
 user_pref("extensions.blocklist.itemURL", "http://%(server)s/extensions-dummy/blocklistItemURL");
--- a/testing/talos/talos/config.py
+++ b/testing/talos/talos/config.py
@@ -95,16 +95,20 @@ DEFAULTS = dict(
         'browser.newtabpage.directory.source':
             '${webserver}/directoryLinks.json',
         'browser.newtabpage.directory.ping': '',
         'browser.newtabpage.introShown': True,
         'browser.safebrowsing.provider.google.gethashURL':
             'http://127.0.0.1/safebrowsing-dummy/gethash',
         'browser.safebrowsing.provider.google.updateURL':
             'http://127.0.0.1/safebrowsing-dummy/update',
+        'browser.safebrowsing.provider.google4.gethashURL':
+            'http://127.0.0.1/safebrowsing4-dummy/gethash',
+        'browser.safebrowsing.provider.google4.updateURL':
+            'http://127.0.0.1/safebrowsing4-dummy/update',
         'browser.safebrowsing.provider.mozilla.gethashURL':
             'http://127.0.0.1/safebrowsing-dummy/gethash',
         'browser.safebrowsing.provider.mozilla.updateURL':
             'http://127.0.0.1/safebrowsing-dummy/update',
         'privacy.trackingprotection.introURL':
             'http://127.0.0.1/trackingprotection/tour',
         'browser.safebrowsing.phishing.enabled': False,
         'browser.safebrowsing.malware.enabled': False,