author | Henry Chang <hchang@mozilla.com> |
Thu, 04 Aug 2016 18:10:06 +0800 | |
changeset 308526 | ae2cbe1419d188ae85ba1d7619f2cf9a1d0f8e4e |
parent 308525 | 06eca66de01d82da85866faca70fad0e7a489fc7 |
child 308527 | 09aabc8742d52b1bcfcea8f6a717f579d90d4250 |
push id | 31131 |
push user | hchang@mozilla.com |
push date | Mon, 08 Aug 2016 02:52:22 +0000 |
treeherder | autoland@09aabc8742d5 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | francois |
bugs | 1274112 |
milestone | 51.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -5128,17 +5128,17 @@ pref("browser.safebrowsing.provider.goog pref("browser.safebrowsing.provider.google.lists", "goog-badbinurl-shavar,goog-downloadwhite-digest256,goog-phish-shavar,googpub-phish-shavar,goog-malware-shavar,goog-unwanted-shavar"); pref("browser.safebrowsing.provider.google.updateURL", "https://safebrowsing.google.com/safebrowsing/downloads?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2&key=%GOOGLE_API_KEY%"); pref("browser.safebrowsing.provider.google.gethashURL", "https://safebrowsing.google.com/safebrowsing/gethash?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2"); pref("browser.safebrowsing.provider.google.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site="); // Prefs for v4. pref("browser.safebrowsing.provider.google4.pver", "4"); pref("browser.safebrowsing.provider.google4.lists", "goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto"); -pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$req=%REQUEST_BASE64%&$ct=application/x-protobuf&key=%GOOGLE_API_KEY%"); +pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%"); pref("browser.safebrowsing.provider.google4.gethashURL", "https://safebrowsing.googleapis.com/v4/fullHashes:find?$req=%REQUEST_BASE64%&$ct=application/x-protobuf&key=%GOOGLE_API_KEY%"); pref("browser.safebrowsing.provider.google4.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site="); pref("browser.safebrowsing.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url="); pref("browser.safebrowsing.reportPhishURL", "https://%LOCALE%.phish-report.mozilla.com/?hl=%LOCALE%&url="); pref("browser.safebrowsing.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url="); // The table and global pref for blocking plugin content
--- a/netwerk/test/unit/test_cookiejars_safebrowsing.js +++ b/netwerk/test/unit/test_cookiejars_safebrowsing.js @@ -110,17 +110,17 @@ add_test(function test_safebrowsing_upda function onUpdateError() { do_throw("ERROR: received onUpdateError!"); } function onDownloadError() { do_throw("ERROR: received onDownloadError!"); } streamUpdater.downloadUpdates("test-phish-simple,test-malware-simple", "", - URL + safebrowsingUpdatePath, onSuccess, onUpdateError, onDownloadError); + true, URL + safebrowsingUpdatePath, onSuccess, onUpdateError, onDownloadError); }); add_test(function test_non_safebrowsing_cookie() { var cookieName = 'regCookie_id0'; var loadContext = new LoadContextCallback(0, false, false, false); function setNonSafeBrowsingCookie() {
--- a/toolkit/components/downloads/test/unit/test_app_rep.js +++ b/toolkit/components/downloads/test/unit/test_app_rep.js @@ -215,16 +215,17 @@ add_test(function test_local_list() { } // Just throw if we ever get an update or download error. function handleError(aEvent) { do_throw("We didn't download or update correctly: " + aEvent); } streamUpdater.downloadUpdates( "goog-downloadwhite-digest256,goog-badbinurl-shavar", "goog-downloadwhite-digest256,goog-badbinurl-shavar;\n", + true, // isPostRequest. "http://localhost:4444/downloads", updateSuccess, handleError, handleError); }); add_test(function test_unlisted() { Services.prefs.setCharPref(appRepURLPref, "http://localhost:4444/download"); let counts = get_telemetry_counts();
--- a/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js +++ b/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js @@ -192,16 +192,17 @@ function waitForUpdates() { // Just throw if we ever get an update or download error. function handleError(aEvent) { do_throw("We didn't download or update correctly: " + aEvent); deferred.reject(); } streamUpdater.downloadUpdates( "goog-downloadwhite-digest256", "goog-downloadwhite-digest256;\n", + true, "http://localhost:4444/downloads", updateSuccess, handleError, handleError); return deferred.promise; } function promiseQueryReputation(query, expectedShouldBlock) { let deferred = Promise.defer(); function onComplete(aShouldBlock, aStatus) {
--- a/toolkit/components/downloads/test/unit/test_app_rep_windows.js +++ b/toolkit/components/downloads/test/unit/test_app_rep_windows.js @@ -292,16 +292,17 @@ function waitForUpdates() { // Just throw if we ever get an update or download error. function handleError(aEvent) { do_throw("We didn't download or update correctly: " + aEvent); deferred.reject(); } streamUpdater.downloadUpdates( "goog-downloadwhite-digest256", "goog-downloadwhite-digest256;\n", + true, "http://localhost:4444/downloads", updateSuccess, handleError, handleError); return deferred.promise; } function promiseQueryReputation(query, expectedShouldBlock) { let deferred = Promise.defer(); function onComplete(aShouldBlock, aStatus) {
--- a/toolkit/components/url-classifier/content/listmanager.js +++ b/toolkit/components/url-classifier/content/listmanager.js @@ -7,17 +7,17 @@ Cu.import("resource://gre/modules/Servic // This is the only implementation of nsIUrlListManager. // A class that manages lists, namely white and black lists for // phishing or malware protection. The ListManager knows how to fetch, // update, and store lists. // // There is a single listmanager for the whole application. // -// TODO more comprehensive update tests, for example add unittest check +// TODO more comprehensive update tests, for example add unittest check // that the listmanagers tables are properly written on updates // Lower and upper limits on the server-provided polling frequency const minDelayMs = 5 * 60 * 1000; const maxDelayMs = 24 * 60 * 60 * 1000; // Log only if browser.safebrowsing.debug is true this.log = function log(...stuff) { @@ -347,91 +347,119 @@ PROT_ListManager.prototype.makeUpdateReq if (!updateUrl) { return; } // An object of the form // { tableList: comma-separated list of tables to request, // tableNames: map of tables that need updating, // request: list of tables and existing chunk ranges from tableData // } - var streamerMap = { tableList: null, tableNames: {}, request: "" }; + var streamerMap = { tableList: null, + tableNames: {}, + requestPayload: "", + isPostRequest: true }; + let useProtobuf = false; + let onceThru = false; for (var tableName in this.tablesData) { // Skip tables not matching this update url if (this.tablesData[tableName].updateUrl != updateUrl) { continue; } // Check if |updateURL| is for 'proto'. (only v4 uses protobuf for now.) // We use the table name 'goog-*-proto' and an additional provider "google4" // to describe the v4 settings. let isCurTableProto = tableName.endsWith('-proto'); - if (useProtobuf && !isCurTableProto) { - log('ERROR: Tables for the same updateURL should all be "proto" or none. ' + - 'Check "browser.safebrowsing.provider.google4.lists"'); + if (!onceThru) { + useProtobuf = isCurTableProto; + onceThru = true; + } else if (useProtobuf !== isCurTableProto) { + log('ERROR: Cannot mix "proto" tables with other types ' + + 'within the same provider.'); } - useProtobuf = isCurTableProto; if (this.needsUpdate_[this.tablesData[tableName].updateUrl][tableName]) { streamerMap.tableNames[tableName] = true; } if (!streamerMap.tableList) { streamerMap.tableList = tableName; } else { streamerMap.tableList += "," + tableName; } } if (useProtobuf) { - // TODO: Bug 1275507 - XPCOM API to build v4 update request. - streamerMap.request = ""; + let tableArray = streamerMap.tableList.split(','); + + // The state is a byte stream which server told us from the + // last table update. The state would be used to do the partial + // update and the empty string means the table has + // never been downloaded. See Bug 1287058 for supporting + // partial update. + let stateArray = []; + tableArray.forEach(() => stateArray.push('')); + + let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + let requestPayload = urlUtils.makeUpdateRequestV4(tableArray, + stateArray, + tableArray.length); + // Use a base64-encoded request. + streamerMap.requestPayload = btoa(requestPayload); + streamerMap.isPostRequest = false; } else { // Build the request. For each table already in the database, include the // chunk data from the database var lines = tableData.split("\n"); for (var i = 0; i < lines.length; i++) { var fields = lines[i].split(";"); var name = fields[0]; if (streamerMap.tableNames[name]) { - streamerMap.request += lines[i] + "\n"; + streamerMap.requestPayload += lines[i] + "\n"; delete streamerMap.tableNames[name]; } } // For each requested table that didn't have chunk data in the database, // request it fresh for (let tableName in streamerMap.tableNames) { - streamerMap.request += tableName + ";\n"; + streamerMap.requestPayload += tableName + ";\n"; } + + streamerMap.isPostRequest = true; } log("update request: " + JSON.stringify(streamerMap, undefined, 2) + "\n"); // Don't send an empty request. - if (streamerMap.request.length > 0) { + if (streamerMap.requestPayload.length > 0) { this.makeUpdateRequestForEntry_(updateUrl, streamerMap.tableList, - streamerMap.request); + streamerMap.requestPayload, + streamerMap.isPostRequest); } else { // We were disabled between kicking off getTables and now. log("Not sending empty request"); } } PROT_ListManager.prototype.makeUpdateRequestForEntry_ = function(updateUrl, tableList, - request) { - log("makeUpdateRequestForEntry_: request " + request + + requestPayload, + isPostRequest) { + log("makeUpdateRequestForEntry_: requestPayload " + requestPayload + " update: " + updateUrl + " tablelist: " + tableList + "\n"); var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"] .getService(Ci.nsIUrlClassifierStreamUpdater); this.requestBackoffs_[updateUrl].noteRequest(); if (!streamer.downloadUpdates( tableList, - request, + requestPayload, + isPostRequest, updateUrl, BindToObject(this.updateSuccess_, this, tableList, updateUrl), BindToObject(this.updateError_, this, tableList, updateUrl), BindToObject(this.downloadError_, this, tableList, updateUrl))) { // Our alarm gets reset in one of the 3 callbacks. log("pending update, queued request until later"); } }
--- a/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl +++ b/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl @@ -15,22 +15,25 @@ interface nsIUrlClassifierStreamUpdater : nsISupports { /** * Try to download updates from updateUrl. If an update is already in * progress, queues the requested update. This is used in nsIUrlListManager * as well as in testing. * @param aRequestTables Comma-separated list of tables included in this * update. - * @param aRequestBody The body for the request. + * @param aRequestPayload The payload for the request. + * @param aIsPostRequest Whether the request should be sent by POST method. + * Should be 'true' for v2 usage. * @param aUpdateUrl The plaintext url from which to request updates. * @param aSuccessCallback Called after a successful update. * @param aUpdateErrorCallback Called for problems applying the update * @param aDownloadErrorCallback Called if we get an http error or a * connection refused error. */ boolean downloadUpdates(in ACString aRequestTables, - in ACString aRequestBody, + in ACString aRequestPayload, + in boolean aIsPostRequest, in ACString aUpdateUrl, in nsIUrlClassifierCallback aSuccessCallback, in nsIUrlClassifierCallback aUpdateErrorCallback, in nsIUrlClassifierCallback aDownloadErrorCallback); };
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp +++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp @@ -98,25 +98,26 @@ nsUrlClassifierStreamUpdater::DownloadDo mDownloadErrorCallback = nullptr; } /////////////////////////////////////////////////////////////////////////////// // nsIUrlClassifierStreamUpdater implementation nsresult nsUrlClassifierStreamUpdater::FetchUpdate(nsIURI *aUpdateUrl, - const nsACString & aRequestBody, + const nsACString & aRequestPayload, + bool aIsPostRequest, const nsACString & aStreamTable) { #ifdef DEBUG { nsCString spec; aUpdateUrl->GetSpec(spec); - LOG(("Fetching update %s from %s", aRequestBody.Data(), spec.get())); + LOG(("Fetching update %s from %s", aRequestPayload.Data(), spec.get())); } #endif nsresult rv; uint32_t loadFlags = nsIChannel::INHIBIT_CACHING | nsIChannel::LOAD_BYPASS_CACHE; rv = NS_NewChannel(getter_AddRefs(mChannel), aUpdateUrl, @@ -129,19 +130,36 @@ nsUrlClassifierStreamUpdater::FetchUpdat NS_ENSURE_SUCCESS(rv, rv); nsCOMPtr<nsILoadInfo> loadInfo = mChannel->GetLoadInfo(); loadInfo->SetOriginAttributes(mozilla::NeckoOriginAttributes(NECKO_SAFEBROWSING_APP_ID, false)); mBeganStream = false; - // If aRequestBody is empty, construct it for the test. - if (!aRequestBody.IsEmpty()) { - rv = AddRequestBody(aRequestBody); + if (!aIsPostRequest) { + // We use POST method to send our request in v2. In v4, the request + // needs to be embedded to the URL and use GET method to send. + // However, from the Chromium source code, a extended HTTP header has + // to be sent along with the request to make the request succeed. + // The following description is from Chromium source code: + // + // "The following header informs the envelope server (which sits in + // front of Google's stubby server) that the received GET request should be + // interpreted as a POST." + // + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("X-HTTP-Method-Override"), + NS_LITERAL_CSTRING("POST"), + false); + NS_ENSURE_SUCCESS(rv, rv); + } else if (!aRequestPayload.IsEmpty()) { + rv = AddRequestBody(aRequestPayload); NS_ENSURE_SUCCESS(rv, rv); } // Set the appropriate content type for file/data URIs, for unit testing // purposes. // This is only used for testing and should be deleted. bool match; if ((NS_SUCCEEDED(aUpdateUrl->SchemeIs("file", &match)) && match) || @@ -171,54 +189,62 @@ nsUrlClassifierStreamUpdater::FetchUpdat mStreamTable = aStreamTable; return NS_OK; } nsresult nsUrlClassifierStreamUpdater::FetchUpdate(const nsACString & aUpdateUrl, - const nsACString & aRequestBody, + const nsACString & aRequestPayload, + bool aIsPostRequest, const nsACString & aStreamTable) { LOG(("(pre) Fetching update from %s\n", PromiseFlatCString(aUpdateUrl).get())); + nsCString updateUrl(aUpdateUrl); + if (!aIsPostRequest) { + updateUrl.AppendPrintf("&$req=%s", nsCString(aRequestPayload).get()); + } + nsCOMPtr<nsIURI> uri; - nsresult rv = NS_NewURI(getter_AddRefs(uri), aUpdateUrl); + nsresult rv = NS_NewURI(getter_AddRefs(uri), updateUrl); NS_ENSURE_SUCCESS(rv, rv); nsAutoCString urlSpec; uri->GetAsciiSpec(urlSpec); LOG(("(post) Fetching update from %s\n", urlSpec.get())); - return FetchUpdate(uri, aRequestBody, aStreamTable); + return FetchUpdate(uri, aRequestPayload, aIsPostRequest, aStreamTable); } NS_IMETHODIMP nsUrlClassifierStreamUpdater::DownloadUpdates( const nsACString &aRequestTables, - const nsACString &aRequestBody, + const nsACString &aRequestPayload, + bool aIsPostRequest, const nsACString &aUpdateUrl, nsIUrlClassifierCallback *aSuccessCallback, nsIUrlClassifierCallback *aUpdateErrorCallback, nsIUrlClassifierCallback *aDownloadErrorCallback, bool *_retval) { NS_ENSURE_ARG(aSuccessCallback); NS_ENSURE_ARG(aUpdateErrorCallback); NS_ENSURE_ARG(aDownloadErrorCallback); if (mIsUpdating) { - LOG(("Already updating, queueing update %s from %s", aRequestBody.Data(), + LOG(("Already updating, queueing update %s from %s", aRequestPayload.Data(), aUpdateUrl.Data())); *_retval = false; PendingRequest *request = mPendingRequests.AppendElement(); request->mTables = aRequestTables; - request->mRequest = aRequestBody; + request->mRequestPayload = aRequestPayload; + request->mIsPostRequest = aIsPostRequest; request->mUrl = aUpdateUrl; request->mSuccessCallback = aSuccessCallback; request->mUpdateErrorCallback = aUpdateErrorCallback; request->mDownloadErrorCallback = aDownloadErrorCallback; return NS_OK; } if (aUpdateUrl.IsEmpty()) { @@ -243,21 +269,22 @@ nsUrlClassifierStreamUpdater::DownloadUp NS_ENSURE_SUCCESS(rv, rv); mInitialized = true; } rv = mDBService->BeginUpdate(this, aRequestTables); if (rv == NS_ERROR_NOT_AVAILABLE) { LOG(("Service busy, already updating, queuing update %s from %s", - aRequestBody.Data(), aUpdateUrl.Data())); + aRequestPayload.Data(), aUpdateUrl.Data())); *_retval = false; PendingRequest *request = mPendingRequests.AppendElement(); request->mTables = aRequestTables; - request->mRequest = aRequestBody; + request->mRequestPayload = aRequestPayload; + request->mIsPostRequest = aIsPostRequest; request->mUrl = aUpdateUrl; request->mSuccessCallback = aSuccessCallback; request->mUpdateErrorCallback = aUpdateErrorCallback; request->mDownloadErrorCallback = aDownloadErrorCallback; return NS_OK; } if (NS_FAILED(rv)) { @@ -267,19 +294,18 @@ nsUrlClassifierStreamUpdater::DownloadUp mSuccessCallback = aSuccessCallback; mUpdateErrorCallback = aUpdateErrorCallback; mDownloadErrorCallback = aDownloadErrorCallback; mIsUpdating = true; *_retval = true; LOG(("FetchUpdate: %s", aUpdateUrl.Data())); - //LOG(("requestBody: %s", aRequestBody.Data())); - return FetchUpdate(aUpdateUrl, aRequestBody, EmptyCString()); + return FetchUpdate(aUpdateUrl, aRequestPayload, aIsPostRequest, EmptyCString()); } /////////////////////////////////////////////////////////////////////////////// // nsIUrlClassifierUpdateObserver implementation NS_IMETHODIMP nsUrlClassifierStreamUpdater::UpdateUrlRequested(const nsACString &aUrl, const nsACString &aTable) @@ -313,17 +339,19 @@ nsresult nsUrlClassifierStreamUpdater::FetchNext() { if (mPendingUpdates.Length() == 0) { return NS_OK; } PendingUpdate &update = mPendingUpdates[0]; LOG(("Fetching update url: %s\n", update.mUrl.get())); - nsresult rv = FetchUpdate(update.mUrl, EmptyCString(), + nsresult rv = FetchUpdate(update.mUrl, + EmptyCString(), + true, // This method is for v2 and v2 is always a POST. update.mTable); if (NS_FAILED(rv)) { LOG(("Error fetching update url: %s\n", update.mUrl.get())); // We can commit the urls that we've applied so far. This is // probably a transient server problem, so trigger backoff. mDownloadErrorCallback->HandleEvent(EmptyCString()); mDownloadError = true; mDBService->FinishUpdate(); @@ -344,17 +372,18 @@ nsUrlClassifierStreamUpdater::FetchNextR } PendingRequest &request = mPendingRequests[0]; LOG(("Stream updater: fetching next request: %s, %s", request.mTables.get(), request.mUrl.get())); bool dummy; DownloadUpdates( request.mTables, - request.mRequest, + request.mRequestPayload, + request.mIsPostRequest, request.mUrl, request.mSuccessCallback, request.mUpdateErrorCallback, request.mDownloadErrorCallback, &dummy); request.mSuccessCallback = nullptr; request.mUpdateErrorCallback = nullptr; request.mDownloadErrorCallback = nullptr;
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h +++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h @@ -49,21 +49,23 @@ private: // Disallow copy constructor nsUrlClassifierStreamUpdater(nsUrlClassifierStreamUpdater&); nsresult AddRequestBody(const nsACString &aRequestBody); // Fetches an update for a single table. nsresult FetchUpdate(nsIURI *aURI, - const nsACString &aRequestBody, + const nsACString &aRequest, + bool aIsPostRequest, const nsACString &aTable); // Dumb wrapper so we don't have to create URIs. nsresult FetchUpdate(const nsACString &aURI, - const nsACString &aRequestBody, + const nsACString &aRequest, + bool aIsPostRequest, const nsACString &aTable); // Fetches the next table, from mPendingUpdates. nsresult FetchNext(); // Fetches the next request, from mPendingRequests nsresult FetchNextRequest(); @@ -73,17 +75,18 @@ private: bool mBeganStream; nsCString mStreamTable; nsCOMPtr<nsIChannel> mChannel; nsCOMPtr<nsIUrlClassifierDBService> mDBService; nsCOMPtr<nsITimer> mTimer; struct PendingRequest { nsCString mTables; - nsCString mRequest; + nsCString mRequestPayload; + bool mIsPostRequest; nsCString mUrl; nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback; nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback; nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback; }; nsTArray<PendingRequest> mPendingRequests; struct PendingUpdate {
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp @@ -206,16 +206,19 @@ nsUrlClassifierUtils::GetKeyForURI(nsIUR static const struct { const char* mListName; uint32_t mThreatType; } THREAT_TYPE_CONV_TABLE[] = { { "goog-malware-proto", MALWARE_THREAT}, // 1 { "googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 { "goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3 { "goog-phish-proto", SOCIAL_ENGINEERING}, // 5 + + // For testing purpose. + { "test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 }; NS_IMETHODIMP nsUrlClassifierUtils::ConvertThreatTypeToListName(uint32_t aThreatType, nsACString& aListName) { for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) {
--- a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js +++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js @@ -195,17 +195,17 @@ function doErrorUpdate(tables, success, */ function doStreamUpdate(updateText, success, failure, downloadFailure) { var dataUpdate = "data:," + encodeURIComponent(updateText); if (!downloadFailure) { downloadFailure = failure; } - streamUpdater.downloadUpdates(allTables, "", + streamUpdater.downloadUpdates(allTables, "", true, dataUpdate, success, failure, downloadFailure); } var gAssertions = { tableData : function(expectedTables, cb) { dbservice.getTables(function(tables) {
--- a/toolkit/components/url-classifier/tests/unit/test_digest256.js +++ b/toolkit/components/url-classifier/tests/unit/test_digest256.js @@ -113,16 +113,17 @@ add_test(function test_update() { // passed back in the callback in nsIUrlClassifierStreamUpdater on success. do_check_eq("1000", aEvent); do_print("All data processed"); run_next_test(); } streamUpdater.downloadUpdates( "goog-downloadwhite-digest256", "goog-downloadwhite-digest256;\n", + true, "http://localhost:4444/downloads", updateSuccess, handleError, handleError); }); add_test(function test_url_not_whitelisted() { let uri = createURI("http://example.com"); let principal = gSecMan.createCodebasePrincipal(uri, {}); gDbService.lookup(principal, "goog-downloadwhite-digest256",
--- a/toolkit/components/url-classifier/tests/unit/test_listmanager.js +++ b/toolkit/components/url-classifier/tests/unit/test_listmanager.js @@ -26,55 +26,61 @@ const TEST_TABLE_DATA_LIST = [ { tableName: "test-listmanager2-digest256", providerName: "google", updateUrl: "http://localhost:4444/safebrowsing/update", gethashUrl: "http://localhost:4444/safebrowsing/gethash2", } ]; -// This table has a different update URL. -const TEST_TABLE_DATA_ANOTHER = { - tableName: "test-listmanageranother-digest256", - providerName: "google", - updateUrl: "http://localhost:5555/safebrowsing/update", - gethashUrl: "http://localhost:5555/safebrowsing/gethash-another", +// This table has a different update URL (for v4). +const TEST_TABLE_DATA_V4 = { + tableName: "test-phish-proto", + providerName: "google4", + updateUrl: "http://localhost:5555/safebrowsing/update?", + gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4", }; const PREF_NEXTUPDATETIME = "browser.safebrowsing.provider.google.nextupdatetime"; +const PREF_NEXTUPDATETIME_V4 = "browser.safebrowsing.provider.google4.nextupdatetime"; let gListManager = Cc["@mozilla.org/url-classifier/listmanager;1"] .getService(Ci.nsIUrlListManager); +let gUrlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + // Global test server for serving safebrowsing updates. let gHttpServ = null; let gUpdateResponse = ""; let gExpectedUpdateRequest = ""; +let gExpectedQueryV4 = ""; -// Handles request for TEST_TABLE_DATA_ANOTHER. -let gHttpServAnother = null; +// Handles request for TEST_TABLE_DATA_V4. +let gHttpServV4 = null; // These two variables are used to synchronize the last two racing updates // (in terms of "update URL") in test_update_all_tables(). let gUpdatedCntForTableData = 0; // For TEST_TABLE_DATA_LIST. -let gIsAnotherUpdated = false; // For TEST_TABLE_DATA_ANOTHER. +let gIsV4Updated = false; // For TEST_TABLE_DATA_V4. prefBranch.setBoolPref("browser.safebrowsing.debug", true); // Register tables. TEST_TABLE_DATA_LIST.forEach(function(t) { gListManager.registerTable(t.tableName, t.providerName, t.updateUrl, t.gethashUrl); }); -gListManager.registerTable(TEST_TABLE_DATA_ANOTHER.tableName, - TEST_TABLE_DATA_ANOTHER.providerName, - TEST_TABLE_DATA_ANOTHER.updateUrl, - TEST_TABLE_DATA_ANOTHER.gethashUrl); + +gListManager.registerTable(TEST_TABLE_DATA_V4.tableName, + TEST_TABLE_DATA_V4.providerName, + TEST_TABLE_DATA_V4.updateUrl, + TEST_TABLE_DATA_V4.gethashUrl); const SERVER_INVOLVED_TEST_CASE_LIST = [ // - Do table0 update. // - Server would respond "a:5:32:32\n[DATA]". function test_update_table0() { disableAllUpdates(); gListManager.enableUpdate(TEST_TABLE_DATA_LIST[0].tableName); @@ -105,41 +111,50 @@ const SERVER_INVOLVED_TEST_CASE_LIST = [ // - Server would respond no chunk control. // // Note that this test MUST be the last one in the array since we rely on // the number of sever-involved test case to synchronize the racing last // two udpates for different URL. function test_update_all_tables() { disableAllUpdates(); - // Enable all tables including TEST_TABLE_DATA_ANOTHER! + // Enable all tables including TEST_TABLE_DATA_V4! TEST_TABLE_DATA_LIST.forEach(function(t) { gListManager.enableUpdate(t.tableName); }); - gListManager.enableUpdate(TEST_TABLE_DATA_ANOTHER.tableName); + gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName); + // Expected results for v2. gExpectedUpdateRequest = TEST_TABLE_DATA_LIST[0].tableName + ";a:5:s:2-12\n" + TEST_TABLE_DATA_LIST[1].tableName + ";\n" + TEST_TABLE_DATA_LIST[2].tableName + ";\n"; gUpdateResponse = "n:1000\n"; + // We test the request against the query string since v4 request + // would be appened to the query string. The request is generated + // by protobuf API (binary) then encoded to base64 format. + let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName], + [""], + 1); + gExpectedQueryV4 = "&$req=" + btoa(requestV4); + forceTableUpdate(); }, ]; SERVER_INVOLVED_TEST_CASE_LIST.forEach(t => add_test(t)); // Tests nsIUrlListManager.getGethashUrl. add_test(function test_getGethashUrl() { TEST_TABLE_DATA_LIST.forEach(function (t) { equal(gListManager.getGethashUrl(t.tableName), t.gethashUrl); }); - equal(gListManager.getGethashUrl(TEST_TABLE_DATA_ANOTHER.tableName), - TEST_TABLE_DATA_ANOTHER.gethashUrl); + equal(gListManager.getGethashUrl(TEST_TABLE_DATA_V4.tableName), + TEST_TABLE_DATA_V4.gethashUrl); run_next_test(); }); function run_test() { // Setup primary testing server. gHttpServ = new HttpServer(); gHttpServ.registerDirectory("/", do_get_cwd()); @@ -160,71 +175,79 @@ function run_test() { if (gUpdatedCntForTableData !== SERVER_INVOLVED_TEST_CASE_LIST.length) { // This is not the last test case so run the next once upon the // the update success. waitForUpdateSuccess(run_next_test); return; } - if (gIsAnotherUpdated) { + if (gIsV4Updated) { run_next_test(); // All tests are done. Just finish. return; } - do_print("Waiting for TEST_TABLE_DATA_ANOTHER to be tested ..."); + do_print("Waiting for TEST_TABLE_DATA_V4 to be tested ..."); }); gHttpServ.start(4444); - // Setup another testing server for the different update URL. - gHttpServAnother = new HttpServer(); - gHttpServAnother.registerDirectory("/", do_get_cwd()); + // Setup v4 testing server for the different update URL. + gHttpServV4 = new HttpServer(); + gHttpServV4.registerDirectory("/", do_get_cwd()); + + gHttpServV4.registerPathHandler("/safebrowsing/update", function(request, response) { + // V4 update request body should be empty. + equal(request.bodyInputStream.available(), 0); - gHttpServAnother.registerPathHandler("/safebrowsing/update", function(request, response) { - let body = NetUtil.readInputStreamToString(request.bodyInputStream, - request.bodyInputStream.available()); + // Not on the spec. Found in Chromium source code... + equal(request.getHeader("X-HTTP-Method-Override"), "POST"); + + // V4 update request uses GET. + equal(request.method, "GET"); - // Verify if the request is as expected. - equal(body, TEST_TABLE_DATA_ANOTHER.tableName + ";\n"); + // V4 append the base64 encoded request to the query string. + equal(request.queryString, gExpectedQueryV4); - // Respond with no chunk control. + // Respond a V2 compatible content for now. In the future we can + // send a meaningful response to test Bug 1284178 to see if the + // update is successfully stored to database. response.setHeader("Content-Type", "application/vnd.google.safebrowsing-update", false); response.setStatusLine(request.httpVersion, 200, "OK"); - let content = "n:1000\n"; response.bodyOutputStream.write(content, content.length); - gIsAnotherUpdated = true; + gIsV4Updated = true; if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) { // All tests are done! run_next_test(); return; } do_print("Wait for all sever-involved tests to be done ..."); }); - gHttpServAnother.start(5555); + gHttpServV4.start(5555); run_next_test(); } // A trick to force updating tables. However, before calling this, we have to // call disableAllUpdates() first to clean up the updateCheckers in listmanager. function forceTableUpdate() { prefBranch.setCharPref(PREF_NEXTUPDATETIME, "1"); + prefBranch.setCharPref(PREF_NEXTUPDATETIME_V4, "1"); gListManager.maybeToggleUpdateChecking(); } function disableAllUpdates() { TEST_TABLE_DATA_LIST.forEach(t => gListManager.disableUpdate(t.tableName)); - gListManager.disableUpdate(TEST_TABLE_DATA_ANOTHER.tableName); + gListManager.disableUpdate(TEST_TABLE_DATA_V4.tableName); } // Since there's no public interface on listmanager to know the update success, // we could only rely on the refresh of "nextupdatetime". function waitForUpdateSuccess(callback) { let nextupdatetime = parseInt(prefBranch.getCharPref(PREF_NEXTUPDATETIME)); do_print("nextupdatetime: " + nextupdatetime); if (nextupdatetime !== 1) { @@ -238,8 +261,16 @@ function waitForUpdateSuccess(callback) function readFileToString(aFilename) { let f = do_get_file(aFilename); let stream = Cc["@mozilla.org/network/file-input-stream;1"] .createInstance(Ci.nsIFileInputStream); stream.init(f, -1, 0, 0); let buf = NetUtil.readInputStreamToString(stream, stream.available()); return buf; } + +function buildUpdateRequestV4InBase64() { + + let request = urlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName], + [""], + 1); + return btoa(request); +}