author | Henry Chang <hchang@mozilla.com> |
Thu, 04 Aug 2016 18:10:12 +0800 | |
changeset 308527 | 09aabc8742d52b1bcfcea8f6a717f579d90d4250 |
parent 308526 | ae2cbe1419d188ae85ba1d7619f2cf9a1d0f8e4e |
child 308528 | 1ed87253fdf1d9e8e6c3a5280a4441cebe7bcf06 |
push id | 31131 |
push user | hchang@mozilla.com |
push date | Mon, 08 Aug 2016 02:52:22 +0000 |
treeherder | autoland@09aabc8742d5 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | francois |
bugs | 1274112 |
milestone | 51.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/toolkit/components/url-classifier/ProtocolParser.cpp +++ b/toolkit/components/url-classifier/ProtocolParser.cpp @@ -6,16 +6,17 @@ #include "ProtocolParser.h" #include "LookupCache.h" #include "nsNetCID.h" #include "mozilla/Logging.h" #include "prnetdb.h" #include "prprf.h" #include "nsUrlClassifierUtils.h" +#include "nsPrintfCString.h" // MOZ_LOG=UrlClassifierProtocolParser:5 mozilla::LazyLogModule gUrlClassifierProtocolParserLog("UrlClassifierProtocolParser"); #define PARSER_LOG(args) MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args) namespace mozilla { namespace safebrowsing { @@ -56,18 +57,18 @@ ParseChunkRange(nsACString::const_iterat *aLast = *aFirst; return true; } return false; } ProtocolParser::ProtocolParser() - : mState(PROTOCOL_STATE_CONTROL) - , mUpdateStatus(NS_OK) + : mUpdateStatus(NS_OK) + , mState(PROTOCOL_STATE_CONTROL) , mUpdateWait(0) , mResetRequested(false) , mTableUpdate(nullptr) { } ProtocolParser::~ProtocolParser() { @@ -109,16 +110,22 @@ ProtocolParser::AppendStream(const nsACS if (NS_FAILED(rv)) { mUpdateStatus = rv; return rv; } } return NS_OK; } +void +ProtocolParser::End() +{ + // Inbound data has already been processed in every AppendStream() call. +} + nsresult ProtocolParser::ProcessControl(bool* aDone) { nsresult rv; nsAutoCString line; *aDone = true; while (NextLine(line)) { @@ -689,10 +696,183 @@ ProtocolParser::GetTableUpdate(const nsA // updates can be transferred to DBServiceWorker, which passes // them back to Classifier when doing the updates, and that // will free them. TableUpdate *update = new TableUpdate(aTable); mTableUpdates.AppendElement(update); return update; } +/////////////////////////////////////////////////////////////////////// +// ProtocolParserProtobuf + +ProtocolParserProtobuf::ProtocolParserProtobuf() +{ +} + +ProtocolParserProtobuf::~ProtocolParserProtobuf() +{ +} + +nsresult +ProtocolParserProtobuf::AppendStream(const nsACString& aData) +{ + // Protobuf data cannot be parsed progressively. Just save the incoming data. + mPending.Append(aData); + return NS_OK; +} + +void +ProtocolParserProtobuf::End() +{ + // mUpdateStatus will be updated to success as long as not all + // the responses are invalid. + mUpdateStatus = NS_ERROR_FAILURE; + + FetchThreatListUpdatesResponse response; + if (!response.ParseFromArray(mPending.get(), mPending.Length())) { + NS_WARNING("ProtocolParserProtobuf failed parsing data."); + return; + } + + for (int i = 0; i < response.list_update_responses_size(); i++) { + auto r = response.list_update_responses(i); + nsresult rv = ProcessOneResponse(r); + if (NS_SUCCEEDED(rv)) { + mUpdateStatus = rv; + } else { + NS_WARNING("Failed to process one response."); + } + } +} + +nsresult +ProtocolParserProtobuf::ProcessOneResponse(const ListUpdateResponse& aResponse) +{ + // A response must have a threat type. + if (!aResponse.has_threat_type()) { + NS_WARNING("Threat type not initialized. This seems to be an invalid response."); + return NS_ERROR_FAILURE; + } + + // Convert threat type to list name. + nsCOMPtr<nsIUrlClassifierUtils> urlUtil = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsCString listName; + nsresult rv = urlUtil->ConvertThreatTypeToListName(aResponse.threat_type(), + listName); + if (NS_FAILED(rv)) { + PARSER_LOG((nsPrintfCString("Threat type to list name conversion error: %d", + aResponse.threat_type())).get()); + return NS_ERROR_FAILURE; + } + + // Test if this is a full update. + bool isFullUpdate = false; + if (aResponse.has_response_type()) { + isFullUpdate = + aResponse.response_type() == ListUpdateResponse::FULL_UPDATE; + } else { + NS_WARNING("Response type not initialized."); + return NS_ERROR_FAILURE; + } + + // Warn if there's no new state. + if (!aResponse.has_new_client_state()) { + NS_WARNING("New state not initialized."); + return NS_ERROR_FAILURE; + } + + PARSER_LOG(("==== Update for threat type '%d' ====", aResponse.threat_type())); + PARSER_LOG(("* listName: %s\n", listName.get())); + PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str())); + PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no"))); + ProcessAdditionOrRemoval(aResponse.additions(), true /*aIsAddition*/); + ProcessAdditionOrRemoval(aResponse.removals(), false); + PARSER_LOG(("\n\n")); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessAdditionOrRemoval(const ThreatEntrySetList& aUpdate, + bool aIsAddition) +{ + nsresult ret = NS_OK; + + for (int i = 0; i < aUpdate.size(); i++) { + auto update = aUpdate.Get(i); + if (!update.has_compression_type()) { + NS_WARNING(nsPrintfCString("%s with no compression type.", + aIsAddition ? "Addition" : "Removal").get()); + continue; + } + + switch (update.compression_type()) { + case COMPRESSION_TYPE_UNSPECIFIED: + NS_WARNING("Unspecified compression type."); + break; + + case RAW: + ret = (aIsAddition ? ProcessRawAddition(update) + : ProcessRawRemoval(update)); + break; + + case RICE: + // Not implemented yet (see bug 1285848), + NS_WARNING("Encoded table update is not supported yet."); + break; + } + } + + return ret; +} + +nsresult +ProtocolParserProtobuf::ProcessRawAddition(const ThreatEntrySet& aAddition) +{ + if (!aAddition.has_raw_hashes()) { + PARSER_LOG(("* No raw addition.")); + return NS_OK; + } + + auto rawHashes = aAddition.raw_hashes(); + if (!rawHashes.has_prefix_size()) { + NS_WARNING("Raw hash has no prefix size"); + return NS_OK; + } + + auto prefixes = rawHashes.raw_hashes(); + if (4 == rawHashes.prefix_size()) { + // Process fixed length prefixes separately. + uint32_t* fixedLengthPrefixes = (uint32_t*)prefixes.c_str(); + size_t numOfFixedLengthPrefixes = prefixes.size() / 4; + PARSER_LOG(("* Raw addition (4 bytes)")); + PARSER_LOG((" - # of prefixes: %d", numOfFixedLengthPrefixes)); + PARSER_LOG((" - Memory address: 0x%p", fixedLengthPrefixes)); + } else { + // TODO: Process variable length prefixes including full hashes. + // See Bug 1283009. + PARSER_LOG((" Raw addition (%d bytes)", rawHashes.prefix_size())); + } + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessRawRemoval(const ThreatEntrySet& aRemoval) +{ + if (!aRemoval.has_raw_indices()) { + NS_WARNING("A removal has no indices."); + return NS_OK; + } + + // indices is an array of int32. + auto indices = aRemoval.raw_indices().indices(); + PARSER_LOG(("* Raw removal")); + PARSER_LOG((" - # of removal: %d", indices.size())); + + return NS_OK; +} + + } // namespace safebrowsing } // namespace mozilla
--- a/toolkit/components/url-classifier/ProtocolParser.h +++ b/toolkit/components/url-classifier/ProtocolParser.h @@ -3,41 +3,46 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef ProtocolParser_h__ #define ProtocolParser_h__ #include "HashStore.h" #include "nsICryptoHMAC.h" +#include "safebrowsing.pb.h" namespace mozilla { namespace safebrowsing { /** - * Some helpers for parsing the safe + * Helpers to parse the "shavar", "digest256" and "simple" list formats. */ class ProtocolParser { public: struct ForwardedUpdate { nsCString table; nsCString url; }; ProtocolParser(); - ~ProtocolParser(); + virtual ~ProtocolParser(); nsresult Status() const { return mUpdateStatus; } nsresult Init(nsICryptoHash* aHasher); void SetCurrentTable(const nsACString& aTable); nsresult Begin(); - nsresult AppendStream(const nsACString& aData); + virtual nsresult AppendStream(const nsACString& aData); + + // Notify that the inbound data is ready for parsing if progressive + // parsing is not supported, for example in V4. + virtual void End(); // Forget the table updates that were created by this pass. It // becomes the caller's responsibility to free them. This is shitty. TableUpdate *GetTableUpdate(const nsACString& aTable); void ForgetTableUpdates() { mTableUpdates.Clear(); } nsTArray<TableUpdate*> &GetTableUpdates() { return mTableUpdates; } // Update information. @@ -68,16 +73,21 @@ private: // contain prefix sizes. nsresult ProcessDigestChunk(const nsACString& aChunk); nsresult ProcessDigestAdd(const nsACString& aChunk); nsresult ProcessDigestSub(const nsACString& aChunk); bool NextLine(nsACString& aLine); void CleanupUpdates(); +protected: + nsCString mPending; + nsresult mUpdateStatus; + +private: enum ParserState { PROTOCOL_STATE_CONTROL, PROTOCOL_STATE_CHUNK }; ParserState mState; enum ChunkType { // Types for shavar tables. @@ -95,25 +105,45 @@ private: uint32_t hashSize; uint32_t length; void Clear() { num = 0; hashSize = 0; length = 0; } }; ChunkState mChunkState; nsCOMPtr<nsICryptoHash> mCryptoHash; - nsresult mUpdateStatus; - nsCString mPending; - uint32_t mUpdateWait; bool mResetRequested; nsTArray<ForwardedUpdate> mForwards; // Keep track of updates to apply before passing them to the DBServiceWorkers. nsTArray<TableUpdate*> mTableUpdates; // Updates to apply to the current table being parsed. TableUpdate *mTableUpdate; }; +// Helpers to parse the "proto" list format. +class ProtocolParserProtobuf final : public ProtocolParser { +public: + typedef FetchThreatListUpdatesResponse_ListUpdateResponse ListUpdateResponse; + typedef google::protobuf::RepeatedPtrField<ThreatEntrySet> ThreatEntrySetList; + +public: + ProtocolParserProtobuf(); + + virtual nsresult AppendStream(const nsACString& aData) override; + virtual void End() override; + +private: + virtual ~ProtocolParserProtobuf(); + + // For parsing update info. + nsresult ProcessOneResponse(const ListUpdateResponse& aResponse); + nsresult ProcessAdditionOrRemoval(const ThreatEntrySetList& aUpdate, + bool aIsAddition); + nsresult ProcessRawAddition(const ThreatEntrySet& aAddition); + nsresult ProcessRawRemoval(const ThreatEntrySet& aRemoval); +}; + } // namespace safebrowsing } // namespace mozilla #endif
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp @@ -436,17 +436,38 @@ nsUrlClassifierDBServiceWorker::BeginStr NS_ENSURE_STATE(mUpdateObserver); NS_ENSURE_STATE(!mInStream); mInStream = true; NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser."); - mProtocolParser = new ProtocolParser(); + // Check if we should use protobuf to parse the update. + bool useProtobuf = false; + for (size_t i = 0; i < mUpdateTables.Length(); i++) { + bool isCurProtobuf = + StringEndsWith(mUpdateTables[i], NS_LITERAL_CSTRING("-proto")); + + if (0 == i) { + // Use the first table name to decice if all the subsequent tables + // should be '-proto'. + useProtobuf = isCurProtobuf; + continue; + } + + if (useProtobuf != isCurProtobuf) { + NS_WARNING("Cannot mix 'proto' tables with other types " + "within the same provider."); + break; + } + } + + mProtocolParser = (useProtobuf ? new ProtocolParserProtobuf() + : new ProtocolParser()); if (!mProtocolParser) return NS_ERROR_OUT_OF_MEMORY; mProtocolParser->Init(mCryptoHash); if (!table.IsEmpty()) { mProtocolParser->SetCurrentTable(table); } @@ -507,16 +528,18 @@ nsUrlClassifierDBServiceWorker::FinishSt return NS_ERROR_NOT_INITIALIZED; } NS_ENSURE_STATE(mInStream); NS_ENSURE_STATE(mUpdateObserver); mInStream = false; + mProtocolParser->End(); + if (NS_SUCCEEDED(mProtocolParser->Status())) { if (mProtocolParser->UpdateWait()) { mUpdateWait = mProtocolParser->UpdateWait(); } // XXX: Only allow forwards from the initial update? const nsTArray<ProtocolParser::ForwardedUpdate> &forwards = mProtocolParser->Forwards(); for (uint32_t i = 0; i < forwards.Length(); i++) {
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.h +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.h @@ -39,17 +39,17 @@ private: // Store the 256 bits in an 8 byte array. uint32_t mMap[8]; }; public: nsUrlClassifierUtils(); - NS_DECL_ISUPPORTS + NS_DECL_THREADSAFE_ISUPPORTS NS_DECL_NSIURLCLASSIFIERUTILS nsresult Init(); nsresult CanonicalizeHostname(const nsACString & hostname, nsACString & _retval); nsresult CanonicalizePath(const nsACString & url, nsACString & _retval);