author | Florian Quèze <florian@queze.net> |
Fri, 10 Jul 2015 21:06:24 +0200 | |
changeset 252362 | 2ae760290f8c388a96fdb38f77f80f3bd9ac94b3 |
parent 252361 | 56ae687cc98a52a83ec003411ace217a7b58d81f |
child 252363 | 1ed69d7f62f297cc86f96102a4d10ee4b8a88645 |
push id | 29034 |
push user | cbook@mozilla.com |
push date | Mon, 13 Jul 2015 09:41:24 +0000 |
treeherder | mozilla-central@167b744bf171 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | markh |
bugs | 1175218 |
milestone | 42.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/browser/app/profile/firefox.js +++ b/browser/app/profile/firefox.js @@ -399,18 +399,23 @@ pref("browser.search.jarURIs", "chrome:/ // pointer to the default engine name pref("browser.search.defaultenginename", "chrome://browser-region/locale/region.properties"); // Ordering of Search Engines in the Engine list. pref("browser.search.order.1", "chrome://browser-region/locale/region.properties"); pref("browser.search.order.2", "chrome://browser-region/locale/region.properties"); pref("browser.search.order.3", "chrome://browser-region/locale/region.properties"); -// Market-specific search defaults (US market only) -pref("browser.search.geoSpecificDefaults", true); +// Market-specific search defaults +// This is disabled globally, and then enabled for individual locales +// in firefox-l10n.js (eg. it's enabled for en-US). +pref("browser.search.geoSpecificDefaults", false); +pref("browser.search.geoSpecificDefaults.url", "https://search.services.mozilla.com/1/%APP%/%VERSION%/%CHANNEL%/%LOCALE%/%REGION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%"); + +// US specific default (used as a fallback if the geoSpecificDefaults request fails). pref("browser.search.defaultenginename.US", "data:text/plain,browser.search.defaultenginename.US=Yahoo"); pref("browser.search.order.US.1", "data:text/plain,browser.search.order.US.1=Yahoo"); pref("browser.search.order.US.2", "data:text/plain,browser.search.order.US.2=Google"); pref("browser.search.order.US.3", "data:text/plain,browser.search.order.US.3=Bing"); // search bar results always open in a new tab pref("browser.search.openintab", false);
--- a/browser/base/content/test/general/browser_contextSearchTabPosition.js +++ b/browser/base/content/test/general/browser_contextSearchTabPosition.js @@ -48,22 +48,23 @@ function test() { ok(provider, "Searches provider is available."); let m = provider.getMeasurement("counts", 3); m.getValues().then(function onValues(data) { let now = new Date(); ok(data.days.hasDay(now), "Have data for today."); let day = data.days.getDay(now); - // Will need to be changed if Yahoo isn't the default search engine. - let defaultProviderID = "yahoo"; + // Will need to be changed if Google isn't the default search engine. + // Note: geoSpecificDefaults are disabled for mochitests, so this is the + // non-US en-US default. + let defaultProviderID = "google"; let field = defaultProviderID + ".contextmenu"; ok(day.has(field), "Have search recorded for context menu."); // If any other mochitests perform a context menu search, this will fail. // The solution will be to look up count at test start and ensure it is // incremented by two. is(day.get(field), 2, "2 searches recorded in FHR."); finish(); }); }); } -
--- a/browser/base/content/test/general/browser_urlbar_search_healthreport.js +++ b/browser/base/content/test/general/browser_urlbar_search_healthreport.js @@ -22,18 +22,20 @@ add_task(function* test_healthreport_sea let provider = reporter.getProvider("org.mozilla.searches"); ok(provider, "Searches provider is available."); let m = provider.getMeasurement("counts", 3); let data = yield m.getValues(); let now = new Date(); let oldCount = 0; - // This will to be need changed if default search engine is not Yahoo. - let defaultEngineID = "yahoo"; + // This will to be need changed if default search engine is not Google. + // Note: geoSpecificDefaults are disabled for mochitests, so this is the + // non-US en-US default. + let defaultEngineID = "google"; let field = defaultEngineID + ".urlbar"; if (data.days.hasDay(now)) { let day = data.days.getDay(now); if (day.has(field)) { oldCount = day.get(field); }
--- a/browser/locales/en-US/firefox-l10n.js +++ b/browser/locales/en-US/firefox-l10n.js @@ -1,7 +1,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. #filter substitution +# LOCALIZATION NOTE: this preference is set to true for en-US specifically, +# locales without this line have the setting set to false by default. +pref("browser.search.geoSpecificDefaults", true); + pref("general.useragent.locale", "@AB_CD@");
--- a/mobile/android/app/mobile.js +++ b/mobile/android/app/mobile.js @@ -260,18 +260,23 @@ pref("browser.search.param.maxSuggestion pref("browser.ssl_override_behavior", 2); pref("browser.xul.error_pages.expert_bad_cert", false); // ordering of search engines in the engine list. pref("browser.search.order.1", "chrome://browser/locale/region.properties"); pref("browser.search.order.2", "chrome://browser/locale/region.properties"); pref("browser.search.order.3", "chrome://browser/locale/region.properties"); -// Market-specific search defaults (US market only) -pref("browser.search.geoSpecificDefaults", true); +// Market-specific search defaults +// This is disabled globally, and then enabled for individual locales +// in firefox-l10n.js (eg. it's enabled for en-US). +pref("browser.search.geoSpecificDefaults", false); +pref("browser.search.geoSpecificDefaults.url", "https://search.services.mozilla.com/1/%APP%/%VERSION%/%CHANNEL%/%LOCALE%/%REGION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%"); + +// US specific default (used as a fallback if the geoSpecificDefaults request fails). pref("browser.search.defaultenginename.US", "chrome://browser/locale/region.properties"); pref("browser.search.order.US.1", "chrome://browser/locale/region.properties"); pref("browser.search.order.US.2", "chrome://browser/locale/region.properties"); pref("browser.search.order.US.3", "chrome://browser/locale/region.properties"); // disable updating pref("browser.search.update", false);
--- a/mobile/android/locales/en-US/mobile-l10n.js +++ b/mobile/android/locales/en-US/mobile-l10n.js @@ -1,7 +1,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. #filter substitution +# LOCALIZATION NOTE: this preference is set to true for en-US specifically, +# locales without this line have the setting set to false by default. +pref("browser.search.geoSpecificDefaults", true); + pref("general.useragent.locale", "@AB_CD@");
--- a/testing/profiles/prefs_general.js +++ b/testing/profiles/prefs_general.js @@ -285,16 +285,18 @@ user_pref("loop.CSP","default-src 'self' // Ensure UITour won't hit the network user_pref("browser.uitour.pinnedTabUrl", "http://%(server)s/uitour-dummy/pinnedTab"); user_pref("browser.uitour.url", "http://%(server)s/uitour-dummy/tour"); // Tell the search service we are running in the US. This also has the desired // side-effect of preventing our geoip lookup. user_pref("browser.search.isUS", true); user_pref("browser.search.countryCode", "US"); +// This will prevent HTTP requests for region defaults. +user_pref("browser.search.geoSpecificDefaults", false); // Make sure the self support tab doesn't hit the network. user_pref("browser.selfsupport.url", "https://%(server)s/selfsupport-dummy/"); user_pref("media.eme.enabled", true); #if defined(XP_WIN) user_pref("media.decoder.heuristic.dormant.timeout", 0);
--- a/toolkit/components/places/tests/unit/test_PlacesSearchAutocompleteProvider.js +++ b/toolkit/components/places/tests/unit/test_PlacesSearchAutocompleteProvider.js @@ -4,16 +4,17 @@ Cu.import("resource://gre/modules/PlacesSearchAutocompleteProvider.jsm"); function run_test() { // Tell the search service we are running in the US. This also has the // desired side-effect of preventing our geoip lookup. Services.prefs.setBoolPref("browser.search.isUS", true); Services.prefs.setCharPref("browser.search.countryCode", "US"); + Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", false); run_next_test(); } add_task(function* search_engine_match() { let engine = yield promiseDefaultSearchEngine(); let token = engine.getResultDomain(); let match = yield PlacesSearchAutocompleteProvider.findMatchByToken(token.substr(0, 1)); do_check_eq(match.url, engine.searchForm);
--- a/toolkit/components/search/nsSearchService.js +++ b/toolkit/components/search/nsSearchService.js @@ -205,16 +205,21 @@ var OS_UNSUPPORTED_PARAMS = [ [OS_PARAM_START_INDEX, OS_PARAM_START_INDEX_DEF], [OS_PARAM_START_PAGE, OS_PARAM_START_PAGE_DEF], ]; // The default engine update interval, in days. This is only used if an engine // specifies an updateURL, but not an updateInterval. const SEARCH_DEFAULT_UPDATE_INTERVAL = 7; +// The default interval before checking again for the name of the +// default engine for the region, in seconds. Only used if the response +// from the server doesn't specify an interval. +const SEARCH_GEO_DEFAULT_UPDATE_INTERVAL = 2592000; // 30 days. + // Returns false for whitespace-only or commented out lines in a // Sherlock file, true otherwise. function isUsefulLine(aLine) { return !(/^\s*($|#)/i.test(aLine)); } this.__defineGetter__("FileUtils", function() { delete this.FileUtils; @@ -403,30 +408,31 @@ loadListener.prototype = { // FIXME: bug 253127 // nsIHttpEventSink onRedirect: function (aChannel, aNewChannel) {}, // nsIProgressEventSink onProgress: function (aRequest, aContext, aProgress, aProgressMax) {}, onStatus: function (aRequest, aContext, aStatus, aStatusArg) {} } -// Method to determine if we should be using geo-specific defaults -function geoSpecificDefaultsEnabled() { - // check to see if this is a partner build. Partner builds should not use geo-specific defaults. - let distroID; +function isPartnerBuild() { try { - distroID = Services.prefs.getCharPref("distribution.id"); + let distroID = Services.prefs.getCharPref("distribution.id"); // Mozilla-provided builds (i.e. funnelcake) are not partner builds if (distroID && !distroID.startsWith("mozilla")) { - return false; + return true; } } catch (e) {} - // if we make it here, the pref should dictate behaviour + return false; +} + +// Method to determine if we should be using geo-specific defaults +function geoSpecificDefaultsEnabled() { let geoSpecificDefaults = false; try { geoSpecificDefaults = Services.prefs.getBoolPref("browser.search.geoSpecificDefaults"); } catch(e) {} return geoSpecificDefaults; } @@ -495,17 +501,17 @@ function getIsUS() { // So we are en-US but have no region pref - fallback to hacky timezone check. let isNA = isUSTimezone(); LOG("getIsUS() fell back to a timezone check with the result=" + isNA); return isNA; } // Helper method to modify preference keys with geo-specific modifiers, if needed. function getGeoSpecificPrefName(basepref) { - if (!geoSpecificDefaultsEnabled()) + if (!geoSpecificDefaultsEnabled() || isPartnerBuild()) return basepref; if (getIsUS()) return basepref + ".US"; return basepref; } // A method that tries to determine if this user is in a US geography. function isUSTimezone() { @@ -527,26 +533,63 @@ function isUSTimezone() { // A less hacky method that tries to determine our country-code via an XHR // geoip lookup. // If this succeeds and we are using an en-US locale, we set the pref used by // the hacky method above, so isUS() can avoid the hacky timezone method. // If it fails we don't touch that pref so isUS() does its normal thing. let ensureKnownCountryCode = Task.async(function* () { // If we have a country-code already stored in our prefs we trust it. + let countryCode; try { - Services.prefs.getCharPref("browser.search.countryCode"); - return; // pref exists, so we've done this before. + countryCode = Services.prefs.getCharPref("browser.search.countryCode"); } catch(e) {} - // We don't have it cached, so fetch it. fetchCountryCode() will call - // storeCountryCode if it gets a result (even if that happens after the - // promise resolves) - yield fetchCountryCode(); + + if (!countryCode) { + // We don't have it cached, so fetch it. fetchCountryCode() will call + // storeCountryCode if it gets a result (even if that happens after the + // promise resolves) and fetchRegionDefault. + yield fetchCountryCode(); + } else { + // if nothing to do, return early. + if (!geoSpecificDefaultsEnabled()) + return; + + let expir = engineMetadataService.getGlobalAttr("searchDefaultExpir") || 0; + if (expir > Date.now()) { + // The territory default we have already fetched hasn't expired yet. + // If we have an engine saved, the hash should be valid, verify it now. + let defaultEngine = engineMetadataService.getGlobalAttr("searchDefault"); + if (!defaultEngine || + engineMetadataService.getGlobalAttr("searchDefaultHash") == getVerificationHash(defaultEngine)) { + // No geo default, or valid hash; nothing to do. + return; + } + } + + yield new Promise(resolve => { + let timeoutMS = Services.prefs.getIntPref("browser.search.geoip.timeout"); + let timerId = setTimeout(() => { + timerId = null; + resolve(); + }, timeoutMS); + + let callback = () => { + clearTimeout(timerId); + resolve(); + }; + fetchRegionDefault().then(callback).catch(err => { + Components.utils.reportError(err); + callback(); + }); + }); + } + // If gInitialized is true then the search service was forced to perform - // a sync initialization during our XHR - capture this via telemetry. + // a sync initialization during our XHRs - capture this via telemetry. Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_CAUSED_SYNC_INIT").add(gInitialized); }); // Store the result of the geoip request as well as any other values and // telemetry which depend on it. function storeCountryCode(cc) { // Set the country-code itself. Services.prefs.setCharPref("browser.search.countryCode", cc); @@ -623,42 +666,59 @@ function fetchCountryCode() { // capture reliable telemetry on what timeout value should actually be // used to ensure most users don't see one while not making it so large // that many users end up doing a sync init of the search service and thus // would see the jank that implies. // (Note we do actually use a timeout on the XHR, but that's set to be a // large value just incase the request never completes - we don't want the // XHR object to live forever) let timeoutMS = Services.prefs.getIntPref("browser.search.geoip.timeout"); + let geoipTimeoutPossible = true; let timerId = setTimeout(() => { LOG("_fetchCountryCode: timeout fetching country information"); - Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(1); + if (geoipTimeoutPossible) + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(1); timerId = null; resolve(); }, timeoutMS); let resolveAndReportSuccess = (result, reason) => { // Even if we timed out, we want to save the country code and everything // related so next startup sees the value and doesn't retry this dance. if (result) { storeCountryCode(result); } Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_RESULT").add(reason); // This notification is just for tests... Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "geoip-lookup-xhr-complete"); - // If we've already timed out then we've already resolved the promise, - // so there's nothing else to do. - if (timerId == null) { - return; + if (timerId) { + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(0); + geoipTimeoutPossible = false; } - Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(0); - clearTimeout(timerId); - resolve(); + + let callback = () => { + // If we've already timed out then we've already resolved the promise, + // so there's nothing else to do. + if (timerId == null) { + return; + } + clearTimeout(timerId); + resolve(); + }; + + if (result && geoSpecificDefaultsEnabled()) { + fetchRegionDefault().then(callback).catch(err => { + Components.utils.reportError(err); + callback(); + }); + } else { + callback(); + } }; let request = new XMLHttpRequest(); // This notification is just for tests... Services.obs.notifyObservers(request, SEARCH_SERVICE_TOPIC, "geoip-lookup-xhr-starting"); request.timeout = 100000; // 100 seconds as the last-chance fallback request.onload = function(event) { let took = Date.now() - startTime; @@ -678,16 +738,133 @@ function fetchCountryCode() { }; request.open("POST", endpoint, true); request.setRequestHeader("Content-Type", "application/json"); request.responseType = "json"; request.send("{}"); }); } +// This will make an HTTP request to a Mozilla server that will return +// JSON data telling us what engine should be set as the default for +// the current region, and how soon we should check again. +// +// The optional cohort value returned by the server is to be kept locally +// and sent to the server the next time we ping it. It lets the server +// identify profiles that have been part of a specific experiment. +// +// This promise may take up to 100s to resolve, it's the caller's +// responsibility to ensure with a timer that we are not going to +// block the async init for too long. +let fetchRegionDefault = () => new Promise(resolve => { + let urlTemplate = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF) + .getCharPref("geoSpecificDefaults.url"); + let endpoint = Services.urlFormatter.formatURL(urlTemplate); + + // As an escape hatch, no endpoint means no region specific defaults. + if (!endpoint) { + resolve(); + return; + } + + // Append the optional cohort value. + const cohortPref = "browser.search.cohort"; + let cohort; + try { + cohort = Services.prefs.getCharPref(cohortPref); + } catch(e) {} + if (cohort) + endpoint += "/" + cohort; + + LOG("fetchRegionDefault starting with endpoint " + endpoint); + + let startTime = Date.now(); + let request = new XMLHttpRequest(); + request.timeout = 100000; // 100 seconds as the last-chance fallback + request.onload = function(event) { + let took = Date.now() - startTime; + + let status = event.target.status; + if (status != 200) { + LOG("fetchRegionDefault failed with HTTP code " + status); + let retryAfter = request.getResponseHeader("retry-after"); + if (retryAfter) { + engineMetadataService.setGlobalAttr("searchDefaultExpir", + Date.now() + retryAfter * 1000); + } + resolve(); + return; + } + + let response = event.target.response || {}; + LOG("received " + response.toSource()); + + if (response.cohort) { + Services.prefs.setCharPref(cohortPref, response.cohort); + } else { + Services.prefs.clearUserPref(cohortPref); + } + + if (response.settings && response.settings.searchDefault) { + let defaultEngine = response.settings.searchDefault; + engineMetadataService.setGlobalAttr("searchDefault", defaultEngine); + let hash = getVerificationHash(defaultEngine); + LOG("fetchRegionDefault saved searchDefault: " + defaultEngine + + " with verification hash: " + hash); + engineMetadataService.setGlobalAttr("searchDefaultHash", hash); + } + + let interval = response.interval || SEARCH_GEO_DEFAULT_UPDATE_INTERVAL; + let milliseconds = interval * 1000; // |interval| is in seconds. + engineMetadataService.setGlobalAttr("searchDefaultExpir", + Date.now() + milliseconds); + + LOG("fetchRegionDefault got success response in " + took + "ms"); + resolve(); + }; + request.ontimeout = function(event) { + LOG("fetchRegionDefault: XHR finally timed-out"); + resolve(); + }; + request.onerror = function(event) { + LOG("fetchRegionDefault: failed to retrieve territory default information"); + resolve(); + }; + request.open("GET", endpoint, true); + request.setRequestHeader("Content-Type", "application/json"); + request.responseType = "json"; + request.send(); +}); + +function getVerificationHash(aName) { + let disclaimer = "By modifying this file, I agree that I am doing so " + + "only within $appName itself, using official, user-driven search " + + "engine selection processes, and in a way which does not circumvent " + + "user consent. I acknowledge that any attempt to change this file " + + "from outside of $appName is a malicious act, and will be responded " + + "to accordingly." + + let salt = OS.Path.basename(OS.Constants.Path.profileDir) + aName + + disclaimer.replace(/\$appName/g, Services.appinfo.name); + + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + + // Data is an array of bytes. + let data = converter.convertToByteArray(salt, {}); + let hasher = Cc["@mozilla.org/security/hash;1"] + .createInstance(Ci.nsICryptoHash); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); +} + + /** * Used to verify a given DOM node's localName and namespaceURI. * @param aElement * The element to verify. * @param aLocalNameArray * An array of strings to compare against aElement's localName. * @param aNameSpaceArray * An array of strings to compare against aElement's namespaceURI. @@ -3363,35 +3540,44 @@ SearchService.prototype = { _engines: { }, __sortedEngines: null, get _sortedEngines() { if (!this.__sortedEngines) return this._buildSortedEngineList(); return this.__sortedEngines; }, - // Get the original Engine object that belongs to the defaultenginename pref - // of the default branch. + // Get the original Engine object that is the default for this region, + // ignoring changes the user may have subsequently made. get _originalDefaultEngine() { - let defaultPrefB = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF); - let nsIPLS = Ci.nsIPrefLocalizedString; - let defaultEngine; - - let defPref = getGeoSpecificPrefName("defaultenginename"); - try { - defaultEngine = defaultPrefB.getComplexValue(defPref, nsIPLS).data; - } catch (ex) { - // If the default pref is invalid (e.g. an add-on set it to a bogus value) - // getEngineByName will just return null, which is the best we can do. + let defaultEngine = engineMetadataService.getGlobalAttr("searchDefault"); + if (defaultEngine && + engineMetadataService.getGlobalAttr("searchDefaultHash") != getVerificationHash(defaultEngine)) { + LOG("get _originalDefaultEngine, invalid searchDefaultHash for: " + defaultEngine); + defaultEngine = ""; } + + if (!defaultEngine) { + let defaultPrefB = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF); + let nsIPLS = Ci.nsIPrefLocalizedString; + + let defPref = getGeoSpecificPrefName("defaultenginename"); + try { + defaultEngine = defaultPrefB.getComplexValue(defPref, nsIPLS).data; + } catch (ex) { + // If the default pref is invalid (e.g. an add-on set it to a bogus value) + // getEngineByName will just return null, which is the best we can do. + } + } + return this.getEngineByName(defaultEngine); }, resetToOriginalDefaultEngine: function SRCH_SVC__resetToOriginalDefaultEngine() { - this.defaultEngine = this._originalDefaultEngine; + this.currentEngine = this._originalDefaultEngine; }, _buildCache: function SRCH_SVC__buildCache() { if (!getBoolPref(BROWSER_SEARCH_PREF + "cache.enabled", true)) return; TelemetryStopwatch.start("SEARCH_SERVICE_BUILD_CACHE_MS"); let cache = {}; @@ -3709,33 +3895,40 @@ SearchService.prototype = { for each (let dir in cache.directories) this._loadEnginesFromCache(dir); LOG("_asyncLoadEngines: done"); }.bind(this)); }, _asyncReInit: function () { + LOG("_asyncReInit"); // Start by clearing the initialized state, so we don't abort early. gInitialized = false; // Clear the engines, too, so we don't stick with the stale ones. this._engines = {}; this.__sortedEngines = null; this._currentEngine = null; this._defaultEngine = null; // Clear the metadata service. engineMetadataService._initialized = false; engineMetadataService._initializer = null; Task.spawn(function* () { try { + LOG("Restarting engineMetadataService"); yield engineMetadataService.init(); - yield this._asyncLoadEngines(); + yield ensureKnownCountryCode(); + + // Due to the HTTP requests done by ensureKnownCountryCode, it's possible that + // at this point a synchronous init has been forced by other code. + if (!gInitialized) + yield this._asyncLoadEngines(); // Typically we'll re-init as a result of a pref observer, // so signal to 'callers' that we're done. Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "reinit-complete"); gInitialized = true; } catch (err) { LOG("Reinit failed: " + err); Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "reinit-failed"); @@ -4290,41 +4483,16 @@ SearchService.prototype = { if (aWithHidden) return this._sortedEngines; return this._sortedEngines.filter(function (engine) { return !engine.hidden; }); }, - _getVerificationHash: function SRCH_SVC__getVerificationHash(aName) { - let disclaimer = "By modifying this file, I agree that I am doing so " + - "only within $appName itself, using official, user-driven search " + - "engine selection processes, and in a way which does not circumvent " + - "user consent. I acknowledge that any attempt to change this file " + - "from outside of $appName is a malicious act, and will be responded " + - "to accordingly." - - let salt = OS.Path.basename(OS.Constants.Path.profileDir) + aName + - disclaimer.replace(/\$appName/g, Services.appinfo.name); - - let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] - .createInstance(Ci.nsIScriptableUnicodeConverter); - converter.charset = "UTF-8"; - - // Data is an array of bytes. - let data = converter.convertToByteArray(salt, {}); - let hasher = Cc["@mozilla.org/security/hash;1"] - .createInstance(Ci.nsICryptoHash); - hasher.init(hasher.SHA256); - hasher.update(data, data.length); - - return hasher.finish(true); - }, - // nsIBrowserSearchService init: function SRCH_SVC_init(observer) { LOG("SearchService.init"); let self = this; if (!this._initStarted) { TelemetryStopwatch.start("SEARCH_SERVICE_INIT_MS"); this._initStarted = true; Task.spawn(function task() { @@ -4668,17 +4836,17 @@ SearchService.prototype = { notifyAction(this._defaultEngine, SEARCH_ENGINE_DEFAULT); }, get currentEngine() { this._ensureInitialized(); if (!this._currentEngine) { let name = engineMetadataService.getGlobalAttr("current"); - if (engineMetadataService.getGlobalAttr("hash") == this._getVerificationHash(name)) { + if (engineMetadataService.getGlobalAttr("hash") == getVerificationHash(name)) { this._currentEngine = this.getEngineByName(name); } } if (!this._currentEngine || this._currentEngine.hidden) this._currentEngine = this._originalDefaultEngine; if (!this._currentEngine || this._currentEngine.hidden) this._currentEngine = this._getSortedEngines(false)[0]; @@ -4708,17 +4876,17 @@ SearchService.prototype = { // build's default engine, so that the currentEngine getter falls back to // whatever the default is. let newName = this._currentEngine.name; if (this._currentEngine == this._originalDefaultEngine) { newName = ""; } engineMetadataService.setGlobalAttr("current", newName); - engineMetadataService.setGlobalAttr("hash", this._getVerificationHash(newName)); + engineMetadataService.setGlobalAttr("hash", getVerificationHash(newName)); notifyAction(this._currentEngine, SEARCH_ENGINE_CURRENT); }, getDefaultEngineInfo() { let result = {}; let engine;
--- a/toolkit/components/search/tests/xpcshell/head_search.js +++ b/toolkit/components/search/tests/xpcshell/head_search.js @@ -246,16 +246,32 @@ function isUSTimezone() { // 600 minutes = 10 hours (UTC-10), which is // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast) let UTCOffset = (new Date()).getTimezoneOffset(); return UTCOffset >= 150 && UTCOffset <= 600; } +const kDefaultenginenamePref = "browser.search.defaultenginename"; +const kTestEngineName = "Test search engine"; +const kLocalePref = "general.useragent.locale"; + +function getDefaultEngineName(isUS) { + const nsIPLS = Ci.nsIPrefLocalizedString; + // Copy the logic from nsSearchService + let pref = kDefaultenginenamePref; + if (isUS === undefined) + isUS = Services.prefs.getCharPref(kLocalePref) == "en-US" && isUSTimezone(); + if (isUS) { + pref += ".US"; + } + return Services.prefs.getComplexValue(pref, nsIPLS).data; +} + /** * Waits for metadata being committed. * @return {Promise} Resolved when the metadata is committed to disk. */ function promiseAfterCommit() { return waitForSearchNotification("write-metadata-to-disk-complete"); } @@ -312,16 +328,18 @@ if (!isChild) { Services.prefs.setBoolPref("browser.search.log", true); // The geo-specific search tests assume certain prefs are already setup, which // might not be true when run in comm-central etc. So create them here. Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", true); Services.prefs.setIntPref("browser.search.geoip.timeout", 2000); // But still disable geoip lookups - tests that need it will re-configure this. Services.prefs.setCharPref("browser.search.geoip.url", ""); + // Also disable region defaults - tests using it will also re-configure it. + Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF).setCharPref("geoSpecificDefaults.url", ""); } /** * After useHttpServer() is called, this string contains the URL of the "data" * directory, including the final slash. */ let gDataUrl; @@ -484,16 +502,34 @@ function waitForSearchNotification(aExpe return; Services.obs.removeObserver(observer, SEARCH_SERVICE_TOPIC); resolve(aSubject); }, SEARCH_SERVICE_TOPIC, false); }); } +function asyncInit() { + return new Promise(resolve => { + Services.search.init(function() { + do_check_true(Services.search.isInitialized); + resolve(); + }); + }); +} + +function asyncReInit() { + let promise = waitForSearchNotification("reinit-complete"); + + Services.search.QueryInterface(Ci.nsIObserver) + .observe(null, "nsPref:changed", kLocalePref); + + return promise; +} + // This "enum" from nsSearchService.js const TELEMETRY_RESULT_ENUM = { SUCCESS: 0, SUCCESS_WITHOUT_DATA: 1, XHRTIMEOUT: 2, ERROR: 3, };
new file mode 100644 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_geodefaults.js @@ -0,0 +1,284 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +var requests = []; +var gServerCohort = ""; + +const kUrlPref = "geoSpecificDefaults.url"; + +const kDayInSeconds = 86400; +const kYearInSeconds = kDayInSeconds * 365; + +function run_test() { + updateAppInfo(); + installTestEngine(); + + let srv = new HttpServer(); + + srv.registerPathHandler("/lookup_defaults", (metadata, response) => { + response.setStatusLine("1.1", 200, "OK"); + let data = {interval: kYearInSeconds, + settings: {searchDefault: "Test search engine"}}; + if (gServerCohort) + data.cohort = gServerCohort; + response.write(JSON.stringify(data)); + requests.push(metadata); + }); + + srv.registerPathHandler("/lookup_fail", (metadata, response) => { + response.setStatusLine("1.1", 404, "Not Found"); + requests.push(metadata); + }); + + srv.registerPathHandler("/lookup_unavailable", (metadata, response) => { + response.setStatusLine("1.1", 503, "Service Unavailable"); + response.setHeader("Retry-After", kDayInSeconds.toString()); + requests.push(metadata); + }); + + srv.start(-1); + do_register_cleanup(() => srv.stop(() => {})); + + let url = "http://localhost:" + srv.identity.primaryPort + "/lookup_defaults?"; + Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF).setCharPref(kUrlPref, url); + // Set a bogus user value so that running the test ensures we ignore it. + Services.prefs.setCharPref(BROWSER_SEARCH_PREF + kUrlPref, "about:blank"); + Services.prefs.setCharPref("browser.search.geoip.url", + 'data:application/json,{"country_code": "FR"}'); + + run_next_test(); +} + +function checkNoRequest() { + do_check_eq(requests.length, 0); +} + +function checkRequest(cohort = "") { + do_check_eq(requests.length, 1); + let req = requests.pop(); + do_check_eq(req._method, "GET"); + do_check_eq(req._queryString, cohort ? "/" + cohort : ""); +} + +function promiseGlobalMetadata() { + return new Promise(resolve => Task.spawn(function* () { + let path = OS.Path.join(OS.Constants.Path.profileDir, "search-metadata.json"); + let bytes = yield OS.File.read(path); + resolve(JSON.parse(new TextDecoder().decode(bytes))["[global]"]); + })); +} + +function promiseSaveGlobalMetadata(globalData) { + return new Promise(resolve => Task.spawn(function* () { + let path = OS.Path.join(OS.Constants.Path.profileDir, "search-metadata.json"); + let bytes = yield OS.File.read(path); + let data = JSON.parse(new TextDecoder().decode(bytes)); + data["[global]"] = globalData; + yield OS.File.writeAtomic(path, + new TextEncoder().encode(JSON.stringify(data))); + resolve(); + })); +} + +let forceExpiration = Task.async(function* () { + let metadata = yield promiseGlobalMetadata(); + + // Make the current geodefaults expire 1s ago. + metadata.searchdefaultexpir = Date.now() - 1000; + yield promiseSaveGlobalMetadata(metadata); +}); + +add_task(function* no_request_if_prefed_off() { + // Disable geoSpecificDefaults and check no HTTP request is made. + Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", false); + yield asyncInit(); + checkNoRequest(); + + // The default engine should be set based on the prefs. + do_check_eq(Services.search.currentEngine.name, getDefaultEngineName(false)); + + // Change the default engine and then revert to default to ensure + // the metadata file exists. + let commitPromise = promiseAfterCommit(); + Services.search.currentEngine = Services.search.getEngineByName(kTestEngineName); + Services.search.resetToOriginalDefaultEngine(); + yield commitPromise; + + // Ensure nothing related to geoSpecificDefaults has been written in the metadata. + let metadata = yield promiseGlobalMetadata(); + do_check_eq(typeof metadata.searchdefaultexpir, "undefined"); + do_check_eq(typeof metadata.searchdefault, "undefined"); + do_check_eq(typeof metadata.searchdefaulthash, "undefined"); + + Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", true); +}); + +add_task(function* should_get_geo_defaults_only_once() { + // (Re)initializing the search service should trigger a request, + // and set the default engine based on it. + let commitPromise = promiseAfterCommit(); + // Due to the previous initialization, we expect the countryCode to already be set. + do_check_true(Services.prefs.prefHasUserValue("browser.search.countryCode")); + do_check_eq(Services.prefs.getCharPref("browser.search.countryCode"), "FR"); + yield asyncReInit(); + checkRequest(); + do_check_eq(Services.search.currentEngine.name, kTestEngineName); + yield commitPromise; + + // Verify the metadata was written correctly. + let metadata = yield promiseGlobalMetadata(); + do_check_eq(typeof metadata.searchdefaultexpir, "number"); + do_check_true(metadata.searchdefaultexpir > Date.now()); + do_check_eq(typeof metadata.searchdefault, "string"); + do_check_eq(metadata.searchdefault, "Test search engine"); + do_check_eq(typeof metadata.searchdefaulthash, "string"); + do_check_eq(metadata.searchdefaulthash.length, 44); + + // The next restart shouldn't trigger a request. + yield asyncReInit(); + checkNoRequest(); + do_check_eq(Services.search.currentEngine.name, kTestEngineName); +}); + +add_task(function* should_request_when_countryCode_not_set() { + Services.prefs.clearUserPref("browser.search.countryCode"); + let commitPromise = promiseAfterCommit(); + yield asyncReInit(); + checkRequest(); + yield commitPromise; +}); + +add_task(function* should_recheck_if_interval_expired() { + yield forceExpiration(); + + let commitPromise = promiseAfterCommit(); + let date = Date.now(); + yield asyncReInit(); + checkRequest(); + yield commitPromise; + + // Check that the expiration timestamp has been updated. + let metadata = yield promiseGlobalMetadata(); + do_check_eq(typeof metadata.searchdefaultexpir, "number"); + do_check_true(metadata.searchdefaultexpir >= date + kYearInSeconds * 1000); + do_check_true(metadata.searchdefaultexpir < date + (kYearInSeconds + 3600) * 1000); +}); + +add_task(function* should_recheck_when_broken_hash() { + // This test verifies both that we ignore saved geo-defaults if the + // hash is invalid, and that we keep the local preferences-based + // default for all of the session in case a synchronous + // initialization was triggered before our HTTP request completed. + + let metadata = yield promiseGlobalMetadata(); + + // Break the hash. + let hash = metadata.searchdefaulthash; + metadata.searchdefaulthash = "broken"; + yield promiseSaveGlobalMetadata(metadata); + + let commitPromise = promiseAfterCommit(); + let reInitPromise = asyncReInit(); + + // Synchronously check the current default engine, to force a sync init. + // The hash is wrong, so we should fallback to the default engine from prefs. + // XXX For some reason forcing a sync init while already asynchronously + // reinitializing causes a shutdown warning related to engineMetadataService's + // finalize method having already been called. Seems harmless for the purpose + // of this test. + do_check_false(Services.search.isInitialized) + do_check_eq(Services.search.currentEngine.name, getDefaultEngineName(false)); + do_check_true(Services.search.isInitialized) + + yield reInitPromise; + checkRequest(); + yield commitPromise; + + // Check that the hash is back to its previous value. + metadata = yield promiseGlobalMetadata(); + do_check_eq(typeof metadata.searchdefaulthash, "string"); + do_check_eq(metadata.searchdefaulthash, hash); + + // The current default engine shouldn't change during a session. + do_check_eq(Services.search.currentEngine.name, getDefaultEngineName(false)); + + // After another restart, the current engine should be back to the geo default, + // without doing yet another request. + yield asyncReInit(); + checkNoRequest(); + do_check_eq(Services.search.currentEngine.name, kTestEngineName); +}); + +add_task(function* should_remember_cohort_id() { + // Check that initially the cohort pref doesn't exist. + const cohortPref = "browser.search.cohort"; + do_check_eq(Services.prefs.getPrefType(cohortPref), Services.prefs.PREF_INVALID); + + // Make the server send a cohort id. + let cohort = gServerCohort = "xpcshell"; + + // Trigger a new request. + yield forceExpiration(); + let commitPromise = promiseAfterCommit(); + yield asyncReInit(); + checkRequest(); + yield commitPromise; + + // Check that the cohort was saved. + do_check_eq(Services.prefs.getPrefType(cohortPref), Services.prefs.PREF_STRING); + do_check_eq(Services.prefs.getCharPref(cohortPref), cohort); + + // Make the server stop sending the cohort. + gServerCohort = ""; + + // Check that the next request sends the previous cohort id, and + // will remove it from the prefs due to the server no longer sending it. + yield forceExpiration(); + commitPromise = promiseAfterCommit(); + yield asyncReInit(); + checkRequest(cohort); + yield commitPromise; + do_check_eq(Services.prefs.getPrefType(cohortPref), Services.prefs.PREF_INVALID); +}); + +add_task(function* should_retry_after_failure() { + let defaultBranch = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF); + let originalUrl = defaultBranch.getCharPref(kUrlPref); + defaultBranch.setCharPref(kUrlPref, originalUrl.replace("defaults", "fail")); + + // Trigger a new request. + yield forceExpiration(); + yield asyncReInit(); + checkRequest(); + + // After another restart, a new request should be triggered automatically without + // the test having to call forceExpiration again. + yield asyncReInit(); + checkRequest(); +}); + +add_task(function* should_honor_retry_after_header() { + let defaultBranch = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF); + let originalUrl = defaultBranch.getCharPref(kUrlPref); + defaultBranch.setCharPref(kUrlPref, originalUrl.replace("fail", "unavailable")); + + // Trigger a new request. + yield forceExpiration(); + let date = Date.now(); + let commitPromise = promiseAfterCommit(); + yield asyncReInit(); + checkRequest(); + yield commitPromise; + + // Check that the expiration timestamp has been updated. + let metadata = yield promiseGlobalMetadata(); + do_check_eq(typeof metadata.searchdefaultexpir, "number"); + do_check_true(metadata.searchdefaultexpir >= date + kDayInSeconds * 1000); + do_check_true(metadata.searchdefaultexpir < date + (kDayInSeconds + 3600) * 1000); + + // After another restart, a new request should not be triggered. + yield asyncReInit(); + checkNoRequest(); +});
--- a/toolkit/components/search/tests/xpcshell/test_selectedEngine.js +++ b/toolkit/components/search/tests/xpcshell/test_selectedEngine.js @@ -1,80 +1,18 @@ /* Any copyright is dedicated to the Public Domain. http://creativecommons.org/publicdomain/zero/1.0/ */ Components.utils.import("resource://gre/modules/osfile.jsm"); -const kDefaultenginenamePref = "browser.search.defaultenginename"; const kSelectedEnginePref = "browser.search.selectedEngine"; -const kTestEngineName = "Test search engine"; - -// These two functions (getLocale and getIsUS) are copied from nsSearchService.js -function getLocale() { - let LOCALE_PREF = "general.useragent.locale"; - return Services.prefs.getCharPref(LOCALE_PREF); -} - -function getIsUS() { - if (getLocale() != "en-US") { - return false; - } - - // Timezone assumptions! We assume that if the system clock's timezone is - // between Newfoundland and Hawaii, that the user is in North America. - - // This includes all of South America as well, but we have relatively few - // en-US users there, so that's OK. - - // 150 minutes = 2.5 hours (UTC-2.5), which is - // Newfoundland Daylight Time (http://www.timeanddate.com/time/zones/ndt) - - // 600 minutes = 10 hours (UTC-10), which is - // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast) - - let UTCOffset = (new Date()).getTimezoneOffset(); - let isNA = UTCOffset >= 150 && UTCOffset <= 600; - - return isNA; -} - -function getDefaultEngineName() { - const nsIPLS = Ci.nsIPrefLocalizedString; - // Copy the logic from nsSearchService - let pref = kDefaultenginenamePref; - if (getIsUS()) { - pref += ".US"; - } - return Services.prefs.getComplexValue(pref, nsIPLS).data; -} - // waitForSearchNotification is in head_search.js let waitForNotification = waitForSearchNotification; -function asyncInit() { - let deferred = Promise.defer(); - - Services.search.init(function() { - do_check_true(Services.search.isInitialized); - deferred.resolve(); - }); - - return deferred.promise; -} - -function asyncReInit() { - let promise = waitForNotification("reinit-complete"); - - Services.search.QueryInterface(Ci.nsIObserver) - .observe(null, "nsPref:changed", "general.useragent.locale"); - - return promise; -} - // Check that the default engine matches the defaultenginename pref add_task(function* test_defaultEngine() { yield asyncInit(); do_check_eq(Services.search.currentEngine.name, getDefaultEngineName()); }); // Giving prefs a user value shouldn't change the selected engine. @@ -133,17 +71,17 @@ add_task(function* test_ignoreInvalidHas let path = OS.Path.join(OS.Constants.Path.profileDir, "search-metadata.json"); let bytes = yield OS.File.read(path); let json = JSON.parse(new TextDecoder().decode(bytes)); // Make the hask invalid. json["[global]"].hash = "invalid"; let data = new TextEncoder().encode(JSON.stringify(json)); - let promise = OS.File.writeAtomic(path, data);//, { tmpPath: path + ".tmp" }); + yield OS.File.writeAtomic(path, data);//, { tmpPath: path + ".tmp" }); // Re-init the search service, and check that the json file is ignored. yield asyncReInit(); do_check_eq(Services.search.currentEngine.name, getDefaultEngineName()); }); // Resetting the engine to the default should remove the saved value. add_task(function* test_settingToDefault() {
--- a/toolkit/components/search/tests/xpcshell/xpcshell.ini +++ b/toolkit/components/search/tests/xpcshell/xpcshell.ini @@ -70,8 +70,9 @@ support-files = [test_sync_addon.js] [test_sync_addon_no_override.js] [test_sync_distribution.js] [test_sync_fallback.js] [test_sync_delay_fallback.js] [test_sync_profile_engine.js] [test_rel_searchform.js] [test_selectedEngine.js] +[test_geodefaults.js]
--- a/toolkit/components/urlformatter/nsURLFormatter.js +++ b/toolkit/components/urlformatter/nsURLFormatter.js @@ -85,16 +85,25 @@ function nsURLFormatterService() { nsURLFormatterService.prototype = { classID: Components.ID("{e6156350-2be8-11db-a98b-0800200c9a66}"), QueryInterface: XPCOMUtils.generateQI([Ci.nsIURLFormatter]), _defaults: { LOCALE: function() Cc["@mozilla.org/chrome/chrome-registry;1"]. getService(Ci.nsIXULChromeRegistry). getSelectedLocale('global'), + REGION: function() { + try { + // When the geoip lookup failed to identify the region, we fallback to + // the 'ZZ' region code to mean 'unknown'. + return Services.prefs.getCharPref("browser.search.region") || "ZZ"; + } catch(e) { + return "ZZ"; + } + }, VENDOR: function() this.appInfo.vendor, NAME: function() this.appInfo.name, ID: function() this.appInfo.ID, VERSION: function() this.appInfo.version, APPBUILDID: function() this.appInfo.appBuildID, PLATFORMVERSION: function() this.appInfo.platformVersion, PLATFORMBUILDID: function() this.appInfo.platformBuildID, APP: function() this.appInfo.name.toLowerCase().replace(/ /, ""),