author | Tiberius Oros <toros@mozilla.com> |
Fri, 20 Jul 2018 12:56:59 +0300 | |
changeset 427448 | 4f12d77b4f9b6adaf06615c1c8cdc14de836dc1a |
parent 427408 | 47f713574cb269d5fb0bbb9bbc908131256ad81c (current diff) |
parent 427447 | f91522b8872496ac750c5549a2ea8f1534c82cfd (diff) |
child 427449 | cd8671232fd851520c6e7a5619b65b420984cfa7 |
child 427459 | 8cce0e7ad6f3f7e91afe13e97e04fc0f0ed400c6 |
child 427507 | 18c964a8c8186475819b42ed3e4b12acd6024c74 |
push id | 34304 |
push user | toros@mozilla.com |
push date | Fri, 20 Jul 2018 09:57:23 +0000 |
treeherder | mozilla-central@4f12d77b4f9b [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | merge |
milestone | 63.0a1 |
first release with | nightly linux32
4f12d77b4f9b
/
63.0a1
/
20180720101508
/
files
nightly linux64
4f12d77b4f9b
/
63.0a1
/
20180720101508
/
files
nightly mac
4f12d77b4f9b
/
63.0a1
/
20180720101508
/
files
nightly win32
4f12d77b4f9b
/
63.0a1
/
20180720101508
/
files
nightly win64
4f12d77b4f9b
/
63.0a1
/
20180720101508
/
files
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
releases | nightly linux32
63.0a1
/
20180720101508
/
pushlog to previous
nightly linux64
63.0a1
/
20180720101508
/
pushlog to previous
nightly mac
63.0a1
/
20180720101508
/
pushlog to previous
nightly win32
63.0a1
/
20180720101508
/
pushlog to previous
nightly win64
63.0a1
/
20180720101508
/
pushlog to previous
|
--- a/browser/base/content/test/static/browser_all_files_referenced.js +++ b/browser/base/content/test/static/browser_all_files_referenced.js @@ -127,16 +127,17 @@ var whitelist = [ {file: "chrome://global/skin/tree/sort-asc-classic.png", platforms: ["linux"]}, {file: "chrome://global/skin/tree/sort-asc.png", platforms: ["linux"]}, {file: "chrome://global/skin/tree/sort-dsc-classic.png", platforms: ["linux"]}, {file: "chrome://global/skin/tree/sort-dsc.png", platforms: ["linux"]}, // Bug 1344267 {file: "chrome://marionette/content/test_anonymous_content.xul"}, {file: "chrome://marionette/content/test_dialog.properties"}, {file: "chrome://marionette/content/test_dialog.xul"}, + {file: "chrome://marionette/content/PerTestCoverageUtils.jsm"}, // Bug 1348533 {file: "chrome://mozapps/skin/downloads/buttons.png", platforms: ["macosx"]}, {file: "chrome://mozapps/skin/downloads/downloadButtons.png", platforms: ["linux", "win"]}, // Bug 1348558 {file: "chrome://mozapps/skin/update/downloadButtons.png", platforms: ["linux"]}, // Bug 1348559 {file: "chrome://pippki/content/resetpassword.xul"},
--- a/dom/base/nsContentUtils.cpp +++ b/dom/base/nsContentUtils.cpp @@ -7764,18 +7764,18 @@ nsContentUtils::IPCTransferableToTransfe aTransferable->SetTransferData(item.flavor().get(), imgPtr, sizeof(nsISupports*)); } else { nsCOMPtr<nsISupportsCString> dataWrapper = do_CreateInstance(NS_SUPPORTS_CSTRING_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); // The buffer contains the terminating null. Shmem itemData = item.data().get_Shmem(); - const nsDependentCString text(itemData.get<char>(), - itemData.Size<char>()); + const nsDependentCSubstring text(itemData.get<char>(), + itemData.Size<char>()); rv = dataWrapper->SetData(text); NS_ENSURE_SUCCESS(rv, rv); rv = aTransferable->SetTransferData(item.flavor().get(), dataWrapper, text.Length()); NS_ENSURE_SUCCESS(rv, rv); } @@ -7946,25 +7946,25 @@ ConvertToShmem(mozilla::dom::nsIContentC { MOZ_ASSERT((aChild && !aParent) || (!aChild && aParent)); IShmemAllocator* allocator = aChild ? static_cast<IShmemAllocator*>(aChild) : static_cast<IShmemAllocator*>(aParent); Shmem result; - if (!allocator->AllocShmem(aInput.Length() + 1, + if (!allocator->AllocShmem(aInput.Length(), SharedMemory::TYPE_BASIC, &result)) { return result; } memcpy(result.get<char>(), aInput.BeginReading(), - aInput.Length() + 1); + aInput.Length()); return result; } void nsContentUtils::TransferableToIPCTransferable(nsITransferable* aTransferable, IPCDataTransfer* aIPCDataTransfer, bool aInSyncMessage,
--- a/dom/base/nsDocument.cpp +++ b/dom/base/nsDocument.cpp @@ -2838,17 +2838,17 @@ nsIDocument::ApplySettingsFromCSP(bool a } } nsresult nsIDocument::InitCSP(nsIChannel* aChannel) { MOZ_ASSERT(!mScriptGlobalObject, "CSP must be initialized before mScriptGlobalObject is set!"); - if (!CSPService::sCSPEnabled) { + if (!StaticPrefs::security_csp_enable()) { MOZ_LOG(gCspPRLog, LogLevel::Debug, ("CSP is disabled, skipping CSP init for document %p", this)); return NS_OK; } nsAutoCString tCspHeaderValue, tCspROHeaderValue; nsCOMPtr<nsIHttpChannel> httpChannel; @@ -12512,18 +12512,19 @@ nsIDocument::MaybeAllowStorageForOpener( } nsAutoString origin; nsresult rv = nsContentUtils::GetUTFOrigin(uri, origin); if (NS_WARN_IF(NS_FAILED(rv))) { return; } - AntiTrackingCommon::AddFirstPartyStorageAccessGrantedFor(origin, - openerInner); + // We don't care when the asynchronous work finishes here. + Unused << AntiTrackingCommon::AddFirstPartyStorageAccessGrantedFor(origin, + openerInner); } bool nsIDocument::HasBeenUserGestureActivated() { if (mUserGestureActivated) { return true; }
--- a/dom/base/nsGlobalWindowOuter.cpp +++ b/dom/base/nsGlobalWindowOuter.cpp @@ -7144,17 +7144,19 @@ nsGlobalWindowOuter::MaybeAllowStorageFo } nsAutoString origin; nsresult rv = nsContentUtils::GetUTFOrigin(aURI, origin); if (NS_WARN_IF(NS_FAILED(rv))) { return; } - AntiTrackingCommon::AddFirstPartyStorageAccessGrantedFor(origin, inner); + // We don't care when the asynchronous work finishes here. + Unused << AntiTrackingCommon::AddFirstPartyStorageAccessGrantedFor(origin, + inner); } //***************************************************************************** // nsGlobalWindowOuter: Helper Functions //***************************************************************************** already_AddRefed<nsIDocShellTreeOwner> nsGlobalWindowOuter::GetTreeOwner()
--- a/dom/html/HTMLMetaElement.cpp +++ b/dom/html/HTMLMetaElement.cpp @@ -89,17 +89,18 @@ HTMLMetaElement::BindToTree(nsIDocument* NS_ENSURE_SUCCESS(rv, rv); if (aDocument && AttrValueIs(kNameSpaceID_None, nsGkAtoms::name, nsGkAtoms::viewport, eIgnoreCase)) { nsAutoString content; GetContent(content); nsContentUtils::ProcessViewportInfo(aDocument, content); } - if (CSPService::sCSPEnabled && aDocument && !aDocument->IsLoadedAsData() && + if (StaticPrefs::security_csp_enable() && aDocument && + !aDocument->IsLoadedAsData() && AttrValueIs(kNameSpaceID_None, nsGkAtoms::httpEquiv, nsGkAtoms::headerCSP, eIgnoreCase)) { // only accept <meta http-equiv="Content-Security-Policy" content=""> if it appears // in the <head> element. Element* headElt = aDocument->GetHeadElement(); if (headElt && nsContentUtils::ContentIsDescendantOf(this, headElt)) { nsAutoString content;
--- a/dom/ipc/ContentChild.cpp +++ b/dom/ipc/ContentChild.cpp @@ -3340,17 +3340,17 @@ ContentChild::RecvInvokeDragSession(nsTA for (uint32_t j = 0; j < items.Length(); ++j) { const IPCDataTransferItem& item = items[j]; RefPtr<nsVariantCC> variant = new nsVariantCC(); if (item.data().type() == IPCDataTransferData::TnsString) { const nsString& data = item.data().get_nsString(); variant->SetAsAString(data); } else if (item.data().type() == IPCDataTransferData::TShmem) { Shmem data = item.data().get_Shmem(); - variant->SetAsACString(nsDependentCString(data.get<char>(), data.Size<char>())); + variant->SetAsACString(nsDependentCSubstring(data.get<char>(), data.Size<char>())); Unused << DeallocShmem(data); } else if (item.data().type() == IPCDataTransferData::TIPCBlob) { RefPtr<BlobImpl> blobImpl = IPCBlobUtils::Deserialize(item.data().get_IPCBlob()); variant->SetAsISupports(blobImpl); } else { continue; }
--- a/dom/ipc/ContentParent.cpp +++ b/dom/ipc/ContentParent.cpp @@ -5782,15 +5782,17 @@ ContentParent::RecvBHRThreadHang(const H obs->NotifyObservers(hangDetails, "bhr-thread-hang", nullptr); } return IPC_OK(); } mozilla::ipc::IPCResult ContentParent::RecvFirstPartyStorageAccessGrantedForOrigin(const Principal& aParentPrincipal, const nsCString& aTrackingOrigin, - const nsCString& aGrantedOrigin) + const nsCString& aGrantedOrigin, + FirstPartyStorageAccessGrantedForOriginResolver&& aResolver) { AntiTrackingCommon::SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(aParentPrincipal, aTrackingOrigin, - aGrantedOrigin); + aGrantedOrigin, + std::move(aResolver)); return IPC_OK(); }
--- a/dom/ipc/ContentParent.h +++ b/dom/ipc/ContentParent.h @@ -1223,17 +1223,18 @@ public: const DiscardedData& aDiscardedData) override; virtual mozilla::ipc::IPCResult RecvBHRThreadHang( const HangDetails& aHangDetails) override; virtual mozilla::ipc::IPCResult RecvFirstPartyStorageAccessGrantedForOrigin(const Principal& aParentPrincipal, const nsCString& aTrackingOrigin, - const nsCString& aGrantedOrigin) override; + const nsCString& aGrantedOrigin, + FirstPartyStorageAccessGrantedForOriginResolver&& aResolver) override; // Notify the ContentChild to enable the input event prioritization when // initializing. void MaybeEnableRemoteInputEventQueue(); public: void SendGetFilesResponseAndForget(const nsID& aID, const GetFilesResponseResult& aResult);
--- a/dom/ipc/PContent.ipdl +++ b/dom/ipc/PContent.ipdl @@ -1160,17 +1160,18 @@ parent: async AddPerformanceMetrics(nsID aID, PerformanceInfo[] aMetrics); /* * A 3rd party tracking origin (aTrackingOrigin) has received the permission * granted to have access to aGrantedOrigin when loaded by aParentPrincipal. */ async FirstPartyStorageAccessGrantedForOrigin(Principal aParentPrincipal, nsCString aTrackingOrigin, - nsCString aGrantedOrigin); + nsCString aGrantedOrigin) + returns (bool unused); both: async AsyncMessage(nsString aMessage, CpowEntry[] aCpows, Principal aPrincipal, ClonedMessageData aData); /** * Notify `push-subscription-modified` observers in the parent and child. */
--- a/dom/ipc/TabParent.cpp +++ b/dom/ipc/TabParent.cpp @@ -3397,17 +3397,17 @@ TabParent::AddInitialDnDDataTo(DataTrans nsContentUtils::DataTransferItemToImage(item, getter_AddRefs(imageContainer)); if (NS_FAILED(rv)) { continue; } variant->SetAsISupports(imageContainer); } else { Shmem data = item.data().get_Shmem(); - variant->SetAsACString(nsDependentCString(data.get<char>(), data.Size<char>())); + variant->SetAsACString(nsDependentCSubstring(data.get<char>(), data.Size<char>())); } mozilla::Unused << DeallocShmem(item.data().get_Shmem()); } // We set aHidden to false, as we don't need to worry about hiding data // from content in the parent process where there is no content. // XXX: Nested Content Processes may change this
--- a/dom/security/nsCSPContext.cpp +++ b/dom/security/nsCSPContext.cpp @@ -315,35 +315,22 @@ NS_IMPL_CLASSINFO(nsCSPContext, nullptr, nsIClassInfo::MAIN_THREAD_ONLY, NS_CSPCONTEXT_CID) NS_IMPL_ISUPPORTS_CI(nsCSPContext, nsIContentSecurityPolicy, nsISerializable) -int32_t nsCSPContext::sScriptSampleMaxLength; -bool nsCSPContext::sViolationEventsEnabled = false; - nsCSPContext::nsCSPContext() : mInnerWindowID(0) , mLoadingContext(nullptr) , mLoadingPrincipal(nullptr) , mQueueUpMessages(true) { - static bool sInitialized = false; - if (!sInitialized) { - Preferences::AddIntVarCache(&sScriptSampleMaxLength, - "security.csp.reporting.script-sample.max-length", - 40); - Preferences::AddBoolVarCache(&sViolationEventsEnabled, - "security.csp.enable_violation_events"); - sInitialized = true; - } - CSPCONTEXTLOG(("nsCSPContext::nsCSPContext")); } nsCSPContext::~nsCSPContext() { CSPCONTEXTLOG(("nsCSPContext::~nsCSPContext")); for (uint32_t i = 0; i < mPolicies.Length(); i++) { delete mPolicies[i]; @@ -1196,17 +1183,17 @@ nsCSPContext::SendReports( return NS_OK; } nsresult nsCSPContext::FireViolationEvent( Element* aTriggeringElement, const mozilla::dom::SecurityPolicyViolationEventInit& aViolationEventInit) { - if (!sViolationEventsEnabled) { + if (!StaticPrefs::security_csp_enable_violation_events()) { return NS_OK; } if (mEventListener) { nsAutoString json; if (aViolationEventInit.ToJSON(json)) { mEventListener->OnCSPViolationEvent(json); }
--- a/dom/security/nsCSPContext.h +++ b/dom/security/nsCSPContext.h @@ -4,16 +4,17 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef nsCSPContext_h___ #define nsCSPContext_h___ #include "mozilla/dom/nsCSPUtils.h" #include "mozilla/dom/SecurityPolicyViolationEvent.h" +#include "mozilla/StaticPrefs.h" #include "nsDataHashtable.h" #include "nsIChannel.h" #include "nsIChannelEventSink.h" #include "nsIClassInfo.h" #include "nsIContentSecurityPolicy.h" #include "nsIInterfaceRequestor.h" #include "nsISerializable.h" #include "nsIStreamListener.h" @@ -135,17 +136,19 @@ class nsCSPContext : public nsIContentSe } nsWeakPtr GetLoadingContext(){ return mLoadingContext; } static uint32_t ScriptSampleMaxLength() { - return std::max(sScriptSampleMaxLength, 0); + return std::max( + mozilla::StaticPrefs::security_csp_reporting_script_sample_max_length(), + 0); } private: bool permitsInternal(CSPDirective aDir, mozilla::dom::Element* aTriggeringElement, nsIURI* aContentLocation, nsIURI* aOriginalURIIfRedirect, const nsAString& aNonce, @@ -160,20 +163,16 @@ class nsCSPContext : public nsIContentSe mozilla::dom::Element* aTriggeringElement, const nsAString& aNonce, const nsAString& aContent, const nsAString& aViolatedDirective, uint32_t aViolatedPolicyIndex, uint32_t aLineNumber, uint32_t aColumnNumber); - static int32_t sScriptSampleMaxLength; - - static bool sViolationEventsEnabled; - nsString mReferrer; uint64_t mInnerWindowID; // used for web console logging nsTArray<nsCSPPolicy*> mPolicies; nsCOMPtr<nsIURI> mSelfURI; nsDataHashtable<nsCStringHashKey, int16_t> mShouldLoadCache; nsCOMPtr<nsILoadGroup> mCallingChannelLoadGroup; nsWeakPtr mLoadingContext; // The CSP hangs off the principal, so let's store a raw pointer of the principal
--- a/dom/security/nsCSPParser.cpp +++ b/dom/security/nsCSPParser.cpp @@ -1,16 +1,17 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/ArrayUtils.h" #include "mozilla/Preferences.h" +#include "mozilla/StaticPrefs.h" #include "nsCOMPtr.h" #include "nsContentUtils.h" #include "nsCSPParser.h" #include "nsCSPUtils.h" #include "nsIConsoleService.h" #include "nsIContentPolicy.h" #include "nsIScriptError.h" #include "nsIStringBundle.h" @@ -56,18 +57,16 @@ static const uint32_t kSubHostPathCharac static const char *const kHashSourceValidFns [] = { "sha256", "sha384", "sha512" }; static const uint32_t kHashSourceValidFnsLen = 3; static const char* const kStyle = "style"; static const char* const kScript = "script"; /* ===== nsCSPParser ==================== */ -bool nsCSPParser::sCSPExperimentalEnabled = false; -bool nsCSPParser::sStrictDynamicEnabled = false; nsCSPParser::nsCSPParser(policyTokens& aTokens, nsIURI* aSelfURI, nsCSPContext* aCSPContext, bool aDeliveredViaMetaTag) : mCurChar(nullptr) , mEndChar(nullptr) , mHasHashOrNonce(false) @@ -79,22 +78,16 @@ nsCSPParser::nsCSPParser(policyTokens& a , mScriptSrc(nullptr) , mParsingFrameAncestorsDir(false) , mTokens(aTokens) , mSelfURI(aSelfURI) , mPolicy(nullptr) , mCSPContext(aCSPContext) , mDeliveredViaMetaTag(aDeliveredViaMetaTag) { - static bool initialized = false; - if (!initialized) { - initialized = true; - Preferences::AddBoolVarCache(&sCSPExperimentalEnabled, "security.csp.experimentalEnabled"); - Preferences::AddBoolVarCache(&sStrictDynamicEnabled, "security.csp.enableStrictDynamic"); - } CSPPARSERLOG(("nsCSPParser::nsCSPParser")); } nsCSPParser::~nsCSPParser() { CSPPARSERLOG(("nsCSPParser::~nsCSPParser")); } @@ -483,17 +476,17 @@ nsCSPParser::keywordSource() } if (CSP_IsKeyword(mCurToken, CSP_REPORT_SAMPLE)) { return new nsCSPKeywordSrc(CSP_UTF16KeywordToEnum(mCurToken)); } if (CSP_IsKeyword(mCurToken, CSP_STRICT_DYNAMIC)) { // make sure strict dynamic is enabled - if (!sStrictDynamicEnabled) { + if (!StaticPrefs::security_csp_enableStrictDynamic()) { return nullptr; } if (!CSP_IsDirective(mCurDir[0], nsIContentSecurityPolicy::SCRIPT_SRC_DIRECTIVE)) { // Todo: Enforce 'strict-dynamic' within default-src; see Bug 1313937 const char16_t* params[] = { u"strict-dynamic" }; logWarningErrorToConsole(nsIScriptError::warningFlag, "ignoringStrictDynamic", params, ArrayLength(params)); return nullptr; @@ -963,17 +956,17 @@ nsCSPDirective* nsCSPParser::directiveName() { CSPPARSERLOG(("nsCSPParser::directiveName, mCurToken: %s, mCurValue: %s", NS_ConvertUTF16toUTF8(mCurToken).get(), NS_ConvertUTF16toUTF8(mCurValue).get())); // Check if it is a valid directive if (!CSP_IsValidDirective(mCurToken) || - (!sCSPExperimentalEnabled && + (!StaticPrefs::security_csp_experimentalEnabled() && CSP_IsDirective(mCurToken, nsIContentSecurityPolicy::REQUIRE_SRI_FOR))) { const char16_t* params[] = { mCurToken.get() }; logWarningErrorToConsole(nsIScriptError::warningFlag, "couldNotProcessUnknownDirective", params, ArrayLength(params)); return nullptr; } // The directive 'reflected-xss' is part of CSP 1.1, see:
--- a/dom/security/nsCSPParser.h +++ b/dom/security/nsCSPParser.h @@ -28,19 +28,16 @@ class nsCSPParser { bool aDeliveredViaMetaTag); private: nsCSPParser(policyTokens& aTokens, nsIURI* aSelfURI, nsCSPContext* aCSPContext, bool aDeliveredViaMetaTag); - static bool sCSPExperimentalEnabled; - static bool sStrictDynamicEnabled; - ~nsCSPParser(); // Parsing the CSP using the source-list from http://www.w3.org/TR/CSP11/#source-list nsCSPPolicy* policy(); void directive(); nsCSPDirective* directiveName(); void directiveValue(nsTArray<nsCSPBaseSrc*>& outSrcs);
--- a/dom/security/nsCSPService.cpp +++ b/dom/security/nsCSPService.cpp @@ -1,41 +1,38 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/Logging.h" +#include "mozilla/Preferences.h" +#include "mozilla/StaticPrefs.h" #include "nsString.h" #include "nsCOMPtr.h" #include "nsIURI.h" #include "nsIPrincipal.h" #include "nsIObserver.h" #include "nsIContent.h" #include "nsCSPService.h" #include "nsIContentSecurityPolicy.h" #include "nsError.h" #include "nsIAsyncVerifyRedirectCallback.h" #include "nsAsyncRedirectVerifyHelper.h" -#include "mozilla/Preferences.h" #include "nsIScriptError.h" #include "nsContentUtils.h" #include "nsContentPolicyUtils.h" using namespace mozilla; -/* Keeps track of whether or not CSP is enabled */ -bool CSPService::sCSPEnabled = true; - static LazyLogModule gCspPRLog("CSP"); CSPService::CSPService() { - Preferences::AddBoolVarCache(&sCSPEnabled, "security.csp.enable"); } CSPService::~CSPService() { mAppStatusCache.Clear(); } NS_IMPL_ISUPPORTS(CSPService, nsIContentPolicy, nsIChannelEventSink) @@ -147,17 +144,18 @@ CSPService::ShouldLoad(nsIURI *aContentL // default decision, CSP can revise it if there's a policy to enforce *aDecision = nsIContentPolicy::ACCEPT; // No need to continue processing if CSP is disabled or if the protocol // or type is *not* subject to CSP. // Please note, the correct way to opt-out of CSP using a custom // protocolHandler is to set one of the nsIProtocolHandler flags // that are whitelistet in subjectToCSP() - if (!sCSPEnabled || !subjectToCSP(aContentLocation, contentType)) { + if (!StaticPrefs::security_csp_enable() || + !subjectToCSP(aContentLocation, contentType)) { return NS_OK; } // Find a principal to retrieve the CSP from. If we don't have a context node // (because, for instance, the load originates in a service worker), or the // requesting principal's CSP overrides our document CSP, use the request // principal. Otherwise, use the document principal. nsCOMPtr<nsINode> node(do_QueryInterface(requestContext)); @@ -277,17 +275,18 @@ CSPService::AsyncOnChannelRedirect(nsICh } // No need to continue processing if CSP is disabled or if the protocol // is *not* subject to CSP. // Please note, the correct way to opt-out of CSP using a custom // protocolHandler is to set one of the nsIProtocolHandler flags // that are whitelistet in subjectToCSP() nsContentPolicyType policyType = loadInfo->InternalContentPolicyType(); - if (!sCSPEnabled || !subjectToCSP(newUri, policyType)) { + if (!StaticPrefs::security_csp_enable() || + !subjectToCSP(newUri, policyType)) { return NS_OK; } /* Since redirecting channels don't call into nsIContentPolicy, we call our * Content Policy implementation directly when redirects occur using the * information set in the LoadInfo when channels are created. * * We check if the CSP permits this host for this type of load, if not,
--- a/dom/security/nsCSPService.h +++ b/dom/security/nsCSPService.h @@ -21,17 +21,16 @@ class CSPService : public nsIContentPoli public nsIChannelEventSink { public: NS_DECL_ISUPPORTS NS_DECL_NSICONTENTPOLICY NS_DECL_NSICHANNELEVENTSINK CSPService(); - static bool sCSPEnabled; protected: virtual ~CSPService(); private: // Maps origins to app status. nsDataHashtable<nsCStringHashKey, uint16_t> mAppStatusCache; };
--- a/dom/workers/ScriptLoader.cpp +++ b/dom/workers/ScriptLoader.cpp @@ -1254,17 +1254,17 @@ private: // properly set the referrer header on fetch/xhr requests. If bug 1340694 // is ever fixed this can be removed. rv = mWorkerPrivate->SetPrincipalFromChannel(channel); NS_ENSURE_SUCCESS(rv, rv); nsCOMPtr<nsIContentSecurityPolicy> csp = mWorkerPrivate->GetCSP(); // We did inherit CSP in bug 1223647. If we do not already have a CSP, we // should get it from the HTTP headers on the worker script. - if (CSPService::sCSPEnabled) { + if (StaticPrefs::security_csp_enable()) { if (!csp) { rv = mWorkerPrivate->SetCSPFromHeaderValues(tCspHeaderValue, tCspROHeaderValue); NS_ENSURE_SUCCESS(rv, rv); } else { csp->EnsureEventTarget(mWorkerPrivate->MainThreadEventTarget()); } }
--- a/dom/workers/WorkerThread.cpp +++ b/dom/workers/WorkerThread.cpp @@ -25,17 +25,22 @@ namespace mozilla { using namespace ipc; namespace dom { namespace { // The C stack size. We use the same stack size on all platforms for // consistency. -const uint32_t kWorkerStackSize = 256 * sizeof(size_t) * 1024; +// +// Note: Our typical equation of 256 machine words works out to 2MB on 64-bit +// platforms. Since that works out to the size of a VM huge page, that can +// sometimes lead to an OS allocating an entire huge page for the stack at once. +// To avoid this, we subtract the size of 2 pages, to be safe. +const uint32_t kWorkerStackSize = 256 * sizeof(size_t) * 1024 - 8192; } // namespace WorkerThreadFriendKey::WorkerThreadFriendKey() { MOZ_COUNT_CTOR(WorkerThreadFriendKey); }
--- a/js/src/frontend/BytecodeEmitter.cpp +++ b/js/src/frontend/BytecodeEmitter.cpp @@ -24,16 +24,17 @@ #include "jsutil.h" #include "ds/Nestable.h" #include "frontend/BytecodeControlStructures.h" #include "frontend/EmitterScope.h" #include "frontend/ForOfLoopControl.h" #include "frontend/IfEmitter.h" #include "frontend/Parser.h" +#include "frontend/SwitchEmitter.h" #include "frontend/TDZCheckCache.h" #include "frontend/TryEmitter.h" #include "vm/BytecodeUtil.h" #include "vm/Debugger.h" #include "vm/GeneratorObject.h" #include "vm/JSAtom.h" #include "vm/JSContext.h" #include "vm/JSFunction.h" @@ -1479,32 +1480,62 @@ BytecodeEmitter::reportError(ParseNode* va_list args; va_start(args, errorNumber); parser->errorReporter().errorAtVA(offset, errorNumber, &args); va_end(args); } +void +BytecodeEmitter::reportError(const Maybe<uint32_t>& maybeOffset, unsigned errorNumber, ...) +{ + MOZ_ASSERT_IF(!maybeOffset, this->scriptStartOffsetSet); + uint32_t offset = maybeOffset ? *maybeOffset : this->scriptStartOffset; + + va_list args; + va_start(args, errorNumber); + + parser->errorReporter().errorAtVA(offset, errorNumber, &args); + + va_end(args); +} + bool BytecodeEmitter::reportExtraWarning(ParseNode* pn, unsigned errorNumber, ...) { MOZ_ASSERT_IF(!pn, this->scriptStartOffsetSet); uint32_t offset = pn ? pn->pn_pos.begin : this->scriptStartOffset; va_list args; va_start(args, errorNumber); bool result = parser->errorReporter().reportExtraWarningErrorNumberVA(nullptr, offset, errorNumber, &args); va_end(args); return result; } bool +BytecodeEmitter::reportExtraWarning(const Maybe<uint32_t>& maybeOffset, + unsigned errorNumber, ...) +{ + MOZ_ASSERT_IF(!maybeOffset, this->scriptStartOffsetSet); + uint32_t offset = maybeOffset ? *maybeOffset : this->scriptStartOffset; + + va_list args; + va_start(args, errorNumber); + + bool result = parser->errorReporter().reportExtraWarningErrorNumberVA(nullptr, offset, errorNumber, &args); + + va_end(args); + return result; +} + +bool BytecodeEmitter::emitNewInit(JSProtoKey key) { const size_t len = 1 + UINT32_INDEX_LEN; ptrdiff_t offset; if (!emitCheck(len, &offset)) return false; jsbytecode* code = this->code(offset); @@ -2306,354 +2337,142 @@ BytecodeEmitter::emitNumberOp(double dva /* * Using MOZ_NEVER_INLINE in here is a workaround for llvm.org/pr14047. * LLVM is deciding to inline this function which uses a lot of stack space * into emitTree which is recursive and uses relatively little stack space. */ MOZ_NEVER_INLINE bool BytecodeEmitter::emitSwitch(ParseNode* pn) { - ParseNode* cases = pn->pn_right; - MOZ_ASSERT(cases->isKind(ParseNodeKind::LexicalScope) || - cases->isKind(ParseNodeKind::StatementList)); - - // Ensure that the column of the switch statement is set properly. - if (!updateSourceCoordNotes(pn->pn_pos.begin)) - return false; - - // Emit code for the discriminant. + ParseNode* lexical = pn->pn_right; + MOZ_ASSERT(lexical->isKind(ParseNodeKind::LexicalScope)); + ParseNode* cases = lexical->scopeBody(); + MOZ_ASSERT(cases->isKind(ParseNodeKind::StatementList)); + + SwitchEmitter se(this); + if (!se.emitDiscriminant(Some(pn->pn_pos.begin))) + return false; if (!emitTree(pn->pn_left)) return false; // Enter the scope before pushing the switch BreakableControl since all // breaks are under this scope. - Maybe<TDZCheckCache> tdzCache; - Maybe<EmitterScope> emitterScope; - if (cases->isKind(ParseNodeKind::LexicalScope)) { - if (!cases->isEmptyScope()) { - tdzCache.emplace(this); - emitterScope.emplace(this); - if (!emitterScope->enterLexical(this, ScopeKind::Lexical, cases->scopeBindings())) - return false; - } - - // Advance |cases| to refer to the switch case list. - cases = cases->scopeBody(); + if (!lexical->isEmptyScope()) { + if (!se.emitLexical(lexical->scopeBindings())) + return false; // A switch statement may contain hoisted functions inside its // cases. The PNX_FUNCDEFS flag is propagated from the STATEMENTLIST // bodies of the cases to the case list. if (cases->pn_xflags & PNX_FUNCDEFS) { - MOZ_ASSERT(emitterScope); for (ParseNode* caseNode = cases->pn_head; caseNode; caseNode = caseNode->pn_next) { if (caseNode->pn_right->pn_xflags & PNX_FUNCDEFS) { if (!emitHoistedFunctionsInList(caseNode->pn_right)) return false; } } } - } - - // After entering the scope, push the switch control. - BreakableControl controlInfo(this, StatementKind::Switch); - - ptrdiff_t top = offset(); - - // Switch bytecodes run from here till end of final case. + } else { + MOZ_ASSERT(!(cases->pn_xflags & PNX_FUNCDEFS)); + } + + SwitchEmitter::TableGenerator tableGen(this); uint32_t caseCount = cases->pn_count; - if (caseCount > JS_BIT(16)) { - reportError(pn, JSMSG_TOO_MANY_CASES); - return false; - } - - // Try for most optimal, fall back if not dense ints. - JSOp switchOp = JSOP_TABLESWITCH; - uint32_t tableLength = 0; - int32_t low, high; - bool hasDefault = false; CaseClause* firstCase = cases->pn_head ? &cases->pn_head->as<CaseClause>() : nullptr; - if (caseCount == 0 || - (caseCount == 1 && (hasDefault = firstCase->isDefault()))) - { - low = 0; - high = -1; + if (caseCount == 0) { + tableGen.finish(0); + } else if (caseCount == 1 && firstCase->isDefault()) { + caseCount = 0; + tableGen.finish(0); } else { - Vector<size_t, 128, SystemAllocPolicy> intmap; - int32_t intmapBitLength = 0; - - low = JSVAL_INT_MAX; - high = JSVAL_INT_MIN; - for (CaseClause* caseNode = firstCase; caseNode; caseNode = caseNode->next()) { if (caseNode->isDefault()) { - hasDefault = true; caseCount--; // one of the "cases" was the default continue; } - if (switchOp == JSOP_CONDSWITCH) + if (tableGen.isInvalid()) continue; - MOZ_ASSERT(switchOp == JSOP_TABLESWITCH); - ParseNode* caseValue = caseNode->caseExpression(); if (caseValue->getKind() != ParseNodeKind::Number) { - switchOp = JSOP_CONDSWITCH; + tableGen.setInvalid(); continue; } int32_t i; if (!NumberEqualsInt32(caseValue->pn_dval, &i)) { - switchOp = JSOP_CONDSWITCH; - continue; - } - - if (unsigned(i + int(JS_BIT(15))) >= unsigned(JS_BIT(16))) { - switchOp = JSOP_CONDSWITCH; - continue; - } - if (i < low) - low = i; - if (i > high) - high = i; - - // Check for duplicates, which require a JSOP_CONDSWITCH. - // We bias i by 65536 if it's negative, and hope that's a rare - // case (because it requires a malloc'd bitmap). - if (i < 0) - i += JS_BIT(16); - if (i >= intmapBitLength) { - size_t newLength = NumWordsForBitArrayOfLength(i + 1); - if (!intmap.resize(newLength)) { - ReportOutOfMemory(cx); - return false; - } - intmapBitLength = newLength * BitArrayElementBits; - } - if (IsBitArrayElementSet(intmap.begin(), intmap.length(), i)) { - switchOp = JSOP_CONDSWITCH; + tableGen.setInvalid(); continue; } - SetBitArrayElement(intmap.begin(), intmap.length(), i); - } - - // Compute table length and select condswitch instead if overlarge or - // more than half-sparse. - if (switchOp == JSOP_TABLESWITCH) { - tableLength = uint32_t(high - low + 1); - if (tableLength >= JS_BIT(16) || tableLength > 2 * caseCount) - switchOp = JSOP_CONDSWITCH; - } - } - - // The note has one or two offsets: first tells total switch code length; - // second (if condswitch) tells offset to first JSOP_CASE. - unsigned noteIndex; - size_t switchSize; - if (switchOp == JSOP_CONDSWITCH) { - // 0 bytes of immediate for unoptimized switch. - switchSize = 0; - if (!newSrcNote3(SRC_CONDSWITCH, 0, 0, ¬eIndex)) + + if (!tableGen.addNumber(i)) + return false; + } + + tableGen.finish(caseCount); + } + + if (!se.validateCaseCount(caseCount)) + return false; + + bool isTableSwitch = tableGen.isValid(); + if (isTableSwitch) { + if (!se.emitTable(tableGen)) return false; } else { - MOZ_ASSERT(switchOp == JSOP_TABLESWITCH); - - // 3 offsets (len, low, high) before the table, 1 per entry. - switchSize = size_t(JUMP_OFFSET_LEN * (3 + tableLength)); - if (!newSrcNote2(SRC_TABLESWITCH, 0, ¬eIndex)) - return false; - } - - // Emit switchOp followed by switchSize bytes of jump or lookup table. - MOZ_ASSERT(top == offset()); - if (!emitN(switchOp, switchSize)) - return false; - - Vector<CaseClause*, 32, SystemAllocPolicy> table; - - JumpList condSwitchDefaultOff; - if (switchOp == JSOP_CONDSWITCH) { - unsigned caseNoteIndex; - bool beforeCases = true; - ptrdiff_t lastCaseOffset = -1; - - // The case conditions need their own TDZ cache since they might not - // all execute. - TDZCheckCache tdzCache(this); + if (!se.emitCond()) + return false; // Emit code for evaluating cases and jumping to case statements. for (CaseClause* caseNode = firstCase; caseNode; caseNode = caseNode->next()) { + if (caseNode->isDefault()) + continue; + ParseNode* caseValue = caseNode->caseExpression(); - // If the expression is a literal, suppress line number emission so // that debugging works more naturally. - if (caseValue) { - if (!emitTree(caseValue, ValueUsage::WantValue, - caseValue->isLiteral() ? SUPPRESS_LINENOTE : EMIT_LINENOTE)) - { - return false; - } - } - - if (!beforeCases) { - // prevCase is the previous JSOP_CASE's bytecode offset. - if (!setSrcNoteOffset(caseNoteIndex, 0, offset() - lastCaseOffset)) - return false; - } - if (!caseValue) { - // This is the default clause. - continue; - } - - if (!newSrcNote2(SRC_NEXTCASE, 0, &caseNoteIndex)) - return false; - - // The case clauses are produced before any of the case body. The - // JumpList is saved on the parsed tree, then later restored and - // patched when generating the cases body. - JumpList caseJump; - if (!emitJump(JSOP_CASE, &caseJump)) - return false; - caseNode->setOffset(caseJump.offset); - lastCaseOffset = caseJump.offset; - - if (beforeCases) { - // Switch note's second offset is to first JSOP_CASE. - unsigned noteCount = notes().length(); - if (!setSrcNoteOffset(noteIndex, 1, lastCaseOffset - top)) - return false; - unsigned noteCountDelta = notes().length() - noteCount; - if (noteCountDelta != 0) - caseNoteIndex += noteCountDelta; - beforeCases = false; - } - } - - // If we didn't have an explicit default (which could fall in between - // cases, preventing us from fusing this setSrcNoteOffset with the call - // in the loop above), link the last case to the implicit default for - // the benefit of IonBuilder. - if (!hasDefault && - !beforeCases && - !setSrcNoteOffset(caseNoteIndex, 0, offset() - lastCaseOffset)) - { - return false; - } - - // Emit default even if no explicit default statement. - if (!emitJump(JSOP_DEFAULT, &condSwitchDefaultOff)) - return false; - } else { - MOZ_ASSERT(switchOp == JSOP_TABLESWITCH); - - // skip default offset. - jsbytecode* pc = code(top + JUMP_OFFSET_LEN); - - // Fill in switch bounds, which we know fit in 16-bit offsets. - SET_JUMP_OFFSET(pc, low); - pc += JUMP_OFFSET_LEN; - SET_JUMP_OFFSET(pc, high); - pc += JUMP_OFFSET_LEN; - - if (tableLength != 0) { - if (!table.growBy(tableLength)) { - ReportOutOfMemory(cx); + if (!emitTree(caseValue, ValueUsage::WantValue, + caseValue->isLiteral() ? SUPPRESS_LINENOTE : EMIT_LINENOTE)) + { return false; } - for (CaseClause* caseNode = firstCase; caseNode; caseNode = caseNode->next()) { - if (ParseNode* caseValue = caseNode->caseExpression()) { - MOZ_ASSERT(caseValue->isKind(ParseNodeKind::Number)); - - int32_t i = int32_t(caseValue->pn_dval); - MOZ_ASSERT(double(i) == caseValue->pn_dval); - - i -= low; - MOZ_ASSERT(uint32_t(i) < tableLength); - MOZ_ASSERT(!table[i]); - table[i] = caseNode; - } - } - } - } - - JumpTarget defaultOffset{ -1 }; + if (!se.emitCaseJump()) + return false; + } + } // Emit code for each case's statements. for (CaseClause* caseNode = firstCase; caseNode; caseNode = caseNode->next()) { - if (switchOp == JSOP_CONDSWITCH && !caseNode->isDefault()) { - // The case offset got saved in the caseNode structure after - // emitting the JSOP_CASE jump instruction above. - JumpList caseCond; - caseCond.offset = caseNode->offset(); - if (!emitJumpTargetAndPatch(caseCond)) - return false; - } - - JumpTarget here; - if (!emitJumpTarget(&here)) - return false; - if (caseNode->isDefault()) - defaultOffset = here; - - // If this is emitted as a TABLESWITCH, we'll need to know this case's - // offset later when emitting the table. Store it in the node's - // pn_offset (giving the field a different meaning vs. how we used it - // on the immediately preceding line of code). - caseNode->setOffset(here.offset); - - TDZCheckCache tdzCache(this); + if (caseNode->isDefault()) { + if (!se.emitDefaultBody()) + return false; + } else { + if (isTableSwitch) { + ParseNode* caseValue = caseNode->caseExpression(); + MOZ_ASSERT(caseValue->isKind(ParseNodeKind::Number)); + + int32_t i = int32_t(caseValue->pn_dval); + MOZ_ASSERT(double(i) == caseValue->pn_dval); + + if (!se.emitCaseBody(i, tableGen)) + return false; + } else { + if (!se.emitCaseBody()) + return false; + } + } if (!emitTree(caseNode->statementList())) return false; } - if (!hasDefault) { - // If no default case, offset for default is to end of switch. - if (!emitJumpTarget(&defaultOffset)) - return false; - } - MOZ_ASSERT(defaultOffset.offset != -1); - - // Set the default offset (to end of switch if no default). - jsbytecode* pc; - if (switchOp == JSOP_CONDSWITCH) { - pc = nullptr; - patchJumpsToTarget(condSwitchDefaultOff, defaultOffset); - } else { - MOZ_ASSERT(switchOp == JSOP_TABLESWITCH); - pc = code(top); - SET_JUMP_OFFSET(pc, defaultOffset.offset - top); - pc += JUMP_OFFSET_LEN; - } - - // Set the SRC_SWITCH note's offset operand to tell end of switch. - if (!setSrcNoteOffset(noteIndex, 0, lastNonJumpTargetOffset() - top)) - return false; - - if (switchOp == JSOP_TABLESWITCH) { - // Skip over the already-initialized switch bounds. - pc += 2 * JUMP_OFFSET_LEN; - - // Fill in the jump table, if there is one. - for (uint32_t i = 0; i < tableLength; i++) { - CaseClause* caseNode = table[i]; - ptrdiff_t off = caseNode ? caseNode->offset() - top : 0; - SET_JUMP_OFFSET(pc, off); - pc += JUMP_OFFSET_LEN; - } - } - - // Patch breaks before leaving the scope, as all breaks are under the - // lexical scope if it exists. - if (!controlInfo.patchBreaks(this)) - return false; - - if (emitterScope && !emitterScope->leave(this)) + if (!se.emitEnd()) return false; return true; } bool BytecodeEmitter::isRunOnceLambda() {
--- a/js/src/frontend/BytecodeEmitter.h +++ b/js/src/frontend/BytecodeEmitter.h @@ -414,17 +414,21 @@ struct MOZ_STACK_CLASS BytecodeEmitter void setScriptStartOffsetIfUnset(TokenPos pos) { if (!scriptStartOffsetSet) { scriptStartOffset = pos.begin; scriptStartOffsetSet = true; } } void reportError(ParseNode* pn, unsigned errorNumber, ...); + void reportError(const mozilla::Maybe<uint32_t>& maybeOffset, + unsigned errorNumber, ...); bool reportExtraWarning(ParseNode* pn, unsigned errorNumber, ...); + bool reportExtraWarning(const mozilla::Maybe<uint32_t>& maybeOffset, + unsigned errorNumber, ...); // If pn contains a useful expression, return true with *answer set to true. // If pn contains a useless expression, return true with *answer set to // false. Return false on error. // // The caller should initialize *answer to false and invoke this function on // an expression statement or similar subtree to decide whether the tree // could produce code that has any side effects. For an expression
--- a/js/src/frontend/ParseNode.h +++ b/js/src/frontend/ParseNode.h @@ -250,26 +250,23 @@ IsTypeofKind(ParseNodeKind kind) * pn_count: 1 + number of formal parameters * pn_tree: ParamsBody or StatementList node * Spread unary pn_kid: expression being spread * * <Statements> * StatementList list pn_head: list of pn_count statements * If ternary pn_kid1: cond, pn_kid2: then, pn_kid3: else or null. * Switch binary pn_left: discriminant - * pn_right: list of Case nodes, with at most one - * default node, or if there are let bindings - * in the top level of the switch body's cases, a - * LexicalScope node that contains the list of - * Case nodes. + * pn_right: LexicalScope node that contains the list + * of Case nodes, with at most one + * default node. * Case binary pn_left: case-expression if CaseClause, or * null if DefaultClause * pn_right: StatementList node for this case's * statements - * pn_u.binary.offset: scratch space for the emitter * While binary pn_left: cond, pn_right: body * DoWhile binary pn_left: body, pn_right: cond * For binary pn_left: either ForIn (for-in statement), * ForOf (for-of) or ForHead (for(;;)) * pn_right: body * ForIn ternary pn_kid1: declaration or expression to left of 'in' * pn_kid2: null * pn_kid3: object expr to right of 'in' @@ -553,17 +550,16 @@ class ParseNode ParseNode* kid3; /* else-part, default case, etc. */ } ternary; struct { /* two kids if binary */ ParseNode* left; ParseNode* right; union { unsigned iflags; /* JSITER_* flags for ParseNodeKind::For node */ bool isStatic; /* only for ParseNodeKind::ClassMethod */ - uint32_t offset; /* for the emitter's use on ParseNodeKind::Case nodes */ }; } binary; struct { /* one kid if unary */ ParseNode* kid; bool prologue; /* directive prologue member (as pn_prologue) */ } unary; struct { /* name, labeled statement, etc. */ @@ -1017,20 +1013,16 @@ class CaseClause : public BinaryNode ParseNode* caseExpression() const { return pn_left; } bool isDefault() const { return !caseExpression(); } ParseNode* statementList() const { return pn_right; } // The next CaseClause in the same switch statement. CaseClause* next() const { return pn_next ? &pn_next->as<CaseClause>() : nullptr; } - // Scratch space used by the emitter. - uint32_t offset() const { return pn_u.binary.offset; } - void setOffset(uint32_t u) { pn_u.binary.offset = u; } - static bool test(const ParseNode& node) { bool match = node.isKind(ParseNodeKind::Case); MOZ_ASSERT_IF(match, node.isArity(PN_BINARY)); MOZ_ASSERT_IF(match, node.isOp(JSOP_NOP)); return match; } };
new file mode 100644 --- /dev/null +++ b/js/src/frontend/SwitchEmitter.cpp @@ -0,0 +1,425 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "frontend/SwitchEmitter.h" + +#include "jsutil.h" + +#include "frontend/BytecodeEmitter.h" +#include "frontend/SharedContext.h" +#include "frontend/SourceNotes.h" +#include "vm/BytecodeUtil.h" +#include "vm/Opcodes.h" +#include "vm/Runtime.h" + +using namespace js; +using namespace js::frontend; + +using mozilla::Maybe; + +bool +SwitchEmitter::TableGenerator::addNumber(int32_t caseValue) +{ + if (isInvalid()) + return true; + + if (unsigned(caseValue + int(JS_BIT(15))) >= unsigned(JS_BIT(16))) { + setInvalid(); + return true; + } + + if (intmap_.isNothing()) + intmap_.emplace(); + + low_ = std::min(low_, caseValue); + high_ = std::max(high_, caseValue); + + // Check for duplicates, which require a JSOP_CONDSWITCH. + // We bias caseValue by 65536 if it's negative, and hope that's a rare case + // (because it requires a malloc'd bitmap). + if (caseValue < 0) + caseValue += JS_BIT(16); + if (caseValue >= intmapBitLength_) { + size_t newLength = NumWordsForBitArrayOfLength(caseValue + 1); + if (!intmap_->resize(newLength)) { + ReportOutOfMemory(bce_->cx); + return false; + } + intmapBitLength_ = newLength * BitArrayElementBits; + } + if (IsBitArrayElementSet(intmap_->begin(), intmap_->length(), caseValue)) { + // Duplicate entry is not supported in table switch. + setInvalid(); + return true; + } + SetBitArrayElement(intmap_->begin(), intmap_->length(), caseValue); + return true; +} + +void +SwitchEmitter::TableGenerator::finish(uint32_t caseCount) +{ + intmap_.reset(); + +#ifdef DEBUG + finished_ = true; +#endif + + if (isInvalid()) + return; + + if (caseCount == 0) { + low_ = 0; + high_ = -1; + return; + } + + // Compute table length and select condswitch instead if overlarge + // or more than half-sparse. + tableLength_ = uint32_t(high_ - low_ + 1); + if (tableLength_ >= JS_BIT(16) || tableLength_ > 2 * caseCount) + setInvalid(); +} + +uint32_t +SwitchEmitter::TableGenerator::toCaseIndex(int32_t caseValue) const +{ + MOZ_ASSERT(finished_); + MOZ_ASSERT(isValid()); + uint32_t caseIndex = uint32_t(caseValue - low_); + MOZ_ASSERT(caseIndex < tableLength_); + return caseIndex; +} + +uint32_t +SwitchEmitter::TableGenerator::tableLength() const +{ + MOZ_ASSERT(finished_); + MOZ_ASSERT(isValid()); + return tableLength_; +} + +SwitchEmitter::SwitchEmitter(BytecodeEmitter* bce) + : bce_(bce) +{} + +bool +SwitchEmitter::emitDiscriminant(const Maybe<uint32_t>& switchPos) +{ + MOZ_ASSERT(state_ == State::Start); + switchPos_ = switchPos; + + if (switchPos_) { + // Ensure that the column of the switch statement is set properly. + if (!bce_->updateSourceCoordNotes(*switchPos_)) + return false; + } + + state_ = State::Discriminant; + return true; +} + +bool +SwitchEmitter::emitLexical(Handle<LexicalScope::Data*> bindings) +{ + MOZ_ASSERT(state_ == State::Discriminant); + MOZ_ASSERT(bindings); + + tdzCacheLexical_.emplace(bce_); + emitterScope_.emplace(bce_); + if (!emitterScope_->enterLexical(bce_, ScopeKind::Lexical, bindings)) + return false; + + state_ = State::Lexical; + return true; +} + +bool +SwitchEmitter::validateCaseCount(uint32_t caseCount) +{ + MOZ_ASSERT(state_ == State::Discriminant || state_ == State::Lexical); + if (caseCount > JS_BIT(16)) { + bce_->reportError(switchPos_, JSMSG_TOO_MANY_CASES); + return false; + } + caseCount_ = caseCount; + + state_ = State::CaseCount; + return true; +} + +bool +SwitchEmitter::emitCond() +{ + MOZ_ASSERT(state_ == State::CaseCount); + + kind_ = Kind::Cond; + + // After entering the scope if necessary, push the switch control. + controlInfo_.emplace(bce_, StatementKind::Switch); + top_ = bce_->offset(); + + if (!caseOffsets_.resize(caseCount_)) { + ReportOutOfMemory(bce_->cx); + return false; + } + + // The note has two offsets: first tells total switch code length; + // second tells offset to first JSOP_CASE. + if (!bce_->newSrcNote3(SRC_CONDSWITCH, 0, 0, ¬eIndex_)) + return false; + + MOZ_ASSERT(top_ == bce_->offset()); + if (!bce_->emitN(JSOP_CONDSWITCH, 0)) + return false; + + tdzCacheCaseAndBody_.emplace(bce_); + + state_ = State::Cond; + return true; +} + +bool +SwitchEmitter::emitTable(const TableGenerator& tableGen) +{ + MOZ_ASSERT(state_ == State::CaseCount); + kind_ = Kind::Table; + + // After entering the scope if necessary, push the switch control. + controlInfo_.emplace(bce_, StatementKind::Switch); + top_ = bce_->offset(); + + // The note has one offset that tells total switch code length. + + // 3 offsets (len, low, high) before the table, 1 per entry. + size_t switchSize = size_t(JUMP_OFFSET_LEN * (3 + tableGen.tableLength())); + if (!bce_->newSrcNote2(SRC_TABLESWITCH, 0, ¬eIndex_)) + return false; + + if (!caseOffsets_.resize(tableGen.tableLength())) { + ReportOutOfMemory(bce_->cx); + return false; + } + + MOZ_ASSERT(top_ == bce_->offset()); + if (!bce_->emitN(JSOP_TABLESWITCH, switchSize)) + return false; + + // Skip default offset. + jsbytecode* pc = bce_->code(top_ + JUMP_OFFSET_LEN); + + // Fill in switch bounds, which we know fit in 16-bit offsets. + SET_JUMP_OFFSET(pc, tableGen.low()); + SET_JUMP_OFFSET(pc + JUMP_OFFSET_LEN, tableGen.high()); + + state_ = State::Table; + return true; +} + +bool +SwitchEmitter::emitCaseOrDefaultJump(uint32_t caseIndex, bool isDefault) +{ + MOZ_ASSERT(kind_ == Kind::Cond); + + if (state_ == State::Case) { + // Link the last JSOP_CASE's SRC_NEXTCASE to current JSOP_CASE or + // JSOP_DEFAULT for the benefit of IonBuilder. + if (!bce_->setSrcNoteOffset(caseNoteIndex_, 0, bce_->offset() - lastCaseOffset_)) + return false; + } + + if (isDefault) { + if (!bce_->emitJump(JSOP_DEFAULT, &condSwitchDefaultOffset_)) + return false; + return true; + } + + if (!bce_->newSrcNote2(SRC_NEXTCASE, 0, &caseNoteIndex_)) + return false; + + JumpList caseJump; + if (!bce_->emitJump(JSOP_CASE, &caseJump)) + return false; + caseOffsets_[caseIndex] = caseJump.offset; + lastCaseOffset_ = caseJump.offset; + + if (state_ == State::Cond) { + // Switch note's second offset is to first JSOP_CASE. + unsigned noteCount = bce_->notes().length(); + if (!bce_->setSrcNoteOffset(noteIndex_, 1, lastCaseOffset_ - top_)) + return false; + unsigned noteCountDelta = bce_->notes().length() - noteCount; + if (noteCountDelta != 0) + caseNoteIndex_ += noteCountDelta; + } + + return true; +} + +bool +SwitchEmitter::emitCaseJump() +{ + MOZ_ASSERT(kind_ == Kind::Cond); + MOZ_ASSERT(state_ == State::Cond || state_ == State::Case); + if (!emitCaseOrDefaultJump(caseIndex_, false)) + return false; + caseIndex_++; + + state_ = State::Case; + return true; +} + +bool +SwitchEmitter::emitImplicitDefault() +{ + MOZ_ASSERT(kind_ == Kind::Cond); + MOZ_ASSERT(state_ == State::Cond || state_ == State::Case); + if (!emitCaseOrDefaultJump(0, true)) + return false; + + caseIndex_ = 0; + + // No internal state after emitting default jump. + return true; +} + +bool +SwitchEmitter::emitCaseBody() +{ + MOZ_ASSERT(kind_ == Kind::Cond); + MOZ_ASSERT(state_ == State::Cond || state_ == State::Case || + state_ == State::CaseBody || state_ == State::DefaultBody); + + tdzCacheCaseAndBody_.reset(); + + if (state_ == State::Cond || state_ == State::Case) { + // For cond switch, JSOP_DEFAULT is always emitted. + if (!emitImplicitDefault()) + return false; + } + + JumpList caseJump; + caseJump.offset = caseOffsets_[caseIndex_]; + if (!bce_->emitJumpTargetAndPatch(caseJump)) + return false; + + JumpTarget here; + if (!bce_->emitJumpTarget(&here)) + return false; + caseIndex_++; + + tdzCacheCaseAndBody_.emplace(bce_); + + state_ = State::CaseBody; + return true; +} + +bool +SwitchEmitter::emitCaseBody(int32_t caseValue, const TableGenerator& tableGen) +{ + MOZ_ASSERT(kind_ == Kind::Table); + MOZ_ASSERT(state_ == State::Table || + state_ == State::CaseBody || state_ == State::DefaultBody); + + tdzCacheCaseAndBody_.reset(); + + JumpTarget here; + if (!bce_->emitJumpTarget(&here)) + return false; + caseOffsets_[tableGen.toCaseIndex(caseValue)] = here.offset; + + tdzCacheCaseAndBody_.emplace(bce_); + + state_ = State::CaseBody; + return true; +} + +bool +SwitchEmitter::emitDefaultBody() +{ + MOZ_ASSERT(state_ == State::Cond || state_ == State::Table || + state_ == State::Case || + state_ == State::CaseBody); + MOZ_ASSERT(!hasDefault_); + + tdzCacheCaseAndBody_.reset(); + + if (state_ == State::Cond || state_ == State::Case) { + // For cond switch, JSOP_DEFAULT is always emitted. + if (!emitImplicitDefault()) + return false; + } + JumpTarget here; + if (!bce_->emitJumpTarget(&here)) + return false; + defaultJumpTargetOffset_ = here; + + tdzCacheCaseAndBody_.emplace(bce_); + + hasDefault_ = true; + state_ = State::DefaultBody; + return true; +} + +bool +SwitchEmitter::emitEnd() +{ + MOZ_ASSERT(state_ == State::Cond || state_ == State::Table || + state_ == State::CaseBody || state_ == State::DefaultBody); + + tdzCacheCaseAndBody_.reset(); + + if (!hasDefault_) { + // If no default case, offset for default is to end of switch. + if (!bce_->emitJumpTarget(&defaultJumpTargetOffset_)) + return false; + } + MOZ_ASSERT(defaultJumpTargetOffset_.offset != -1); + + // Set the default offset (to end of switch if no default). + jsbytecode* pc; + if (kind_ == Kind::Cond) { + pc = nullptr; + bce_->patchJumpsToTarget(condSwitchDefaultOffset_, defaultJumpTargetOffset_); + } else { + // Fill in the default jump target. + pc = bce_->code(top_); + SET_JUMP_OFFSET(pc, defaultJumpTargetOffset_.offset - top_); + pc += JUMP_OFFSET_LEN; + } + + // Set the SRC_SWITCH note's offset operand to tell end of switch. + if (!bce_->setSrcNoteOffset(noteIndex_, 0, bce_->lastNonJumpTargetOffset() - top_)) + return false; + + if (kind_ == Kind::Table) { + // Skip over the already-initialized switch bounds. + pc += 2 * JUMP_OFFSET_LEN; + + // Fill in the jump table, if there is one. + for (uint32_t i = 0, length = caseOffsets_.length(); i < length; i++) { + ptrdiff_t off = caseOffsets_[i]; + SET_JUMP_OFFSET(pc, off == 0 ? 0 : off - top_); + pc += JUMP_OFFSET_LEN; + } + } + + // Patch breaks before leaving the scope, as all breaks are under the + // lexical scope if it exists. + if (!controlInfo_->patchBreaks(bce_)) + return false; + + if (emitterScope_ && !emitterScope_->leave(bce_)) + return false; + + emitterScope_.reset(); + tdzCacheLexical_.reset(); + + controlInfo_.reset(); + + state_ = State::End; + return true; +}
new file mode 100644 --- /dev/null +++ b/js/src/frontend/SwitchEmitter.h @@ -0,0 +1,469 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef frontend_SwitchEmitter_h +#define frontend_SwitchEmitter_h + +#include "mozilla/Attributes.h" +#include "mozilla/Maybe.h" + +#include <stddef.h> +#include <stdint.h> + +#include "frontend/BytecodeControlStructures.h" +#include "frontend/EmitterScope.h" +#include "frontend/JumpList.h" +#include "frontend/TDZCheckCache.h" +#include "gc/Rooting.h" +#include "js/AllocPolicy.h" +#include "js/Value.h" +#include "js/Vector.h" +#include "vm/Scope.h" + +namespace js { +namespace frontend { + +struct BytecodeEmitter; + +// Class for emitting bytecode for switch-case-default block. +// +// Usage: (check for the return value is omitted for simplicity) +// +// `switch (discriminant) { case c1_expr: c1_body; }` +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// +// se.validateCaseCount(1); +// se.emitCond(); +// +// emit(c1_expr); +// se.emitCaseJump(); +// +// se.emitCaseBody(); +// emit(c1_body); +// +// se.emitEnd(); +// +// `switch (discriminant) { case c1_expr: c1_body; case c2_expr: c2_body; +// default: def_body; }` +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// +// se.validateCaseCount(2); +// se.emitCond(); +// +// emit(c1_expr); +// se.emitCaseJump(); +// +// emit(c2_expr); +// se.emitCaseJump(); +// +// se.emitCaseBody(); +// emit(c1_body); +// +// se.emitCaseBody(); +// emit(c2_body); +// +// se.emitDefaultBody(); +// emit(def_body); +// +// se.emitEnd(); +// +// `switch (discriminant) { case c1_expr: c1_body; case c2_expr: c2_body; }` +// with Table Switch +// SwitchEmitter::TableGenerator tableGen(this); +// tableGen.addNumber(c1_expr_value); +// tableGen.addNumber(c2_expr_value); +// tableGen.finish(2); +// +// // If `!tableGen.isValid()` here, `emitCond` should be used instead. +// +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// se.validateCaseCount(2); +// se.emitTable(tableGen); +// +// se.emitCaseBody(c1_expr_value, tableGen); +// emit(c1_body); +// +// se.emitCaseBody(c2_expr_value, tableGen); +// emit(c2_body); +// +// se.emitEnd(); +// +// `switch (discriminant) { case c1_expr: c1_body; case c2_expr: c2_body; +// default: def_body; }` +// with Table Switch +// SwitchEmitter::TableGenerator tableGen(bce); +// tableGen.addNumber(c1_expr_value); +// tableGen.addNumber(c2_expr_value); +// tableGen.finish(2); +// +// // If `!tableGen.isValid()` here, `emitCond` should be used instead. +// +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// se.validateCaseCount(2); +// se.emitTable(tableGen); +// +// se.emitCaseBody(c1_expr_value, tableGen); +// emit(c1_body); +// +// se.emitCaseBody(c2_expr_value, tableGen); +// emit(c2_body); +// +// se.emitDefaultBody(); +// emit(def_body); +// +// se.emitEnd(); +// +// `switch (discriminant) { case c1_expr: c1_body; }` +// in case c1_body contains lexical bindings +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// +// se.validateCaseCount(1); +// +// se.emitLexical(bindings); +// +// se.emitCond(); +// +// emit(c1_expr); +// se.emitCaseJump(); +// +// se.emitCaseBody(); +// emit(c1_body); +// +// se.emitEnd(); +// +// `switch (discriminant) { case c1_expr: c1_body; }` +// in case c1_body contains hosted functions +// SwitchEmitter se(this); +// se.emitDiscriminant(Some(offset_of_switch)); +// emit(discriminant); +// +// se.validateCaseCount(1); +// +// se.emitLexical(bindings); +// emit(hosted functions); +// +// se.emitCond(); +// +// emit(c1_expr); +// se.emitCaseJump(); +// +// se.emitCaseBody(); +// emit(c1_body); +// +// se.emitEnd(); +// +class MOZ_STACK_CLASS SwitchEmitter +{ + // Bytecode for each case. + // + // Cond Switch + // {discriminant} + // JSOP_CONDSWITCH + // + // {c1_expr} + // JSOP_CASE c1 + // + // JSOP_JUMPTARGET + // {c2_expr} + // JSOP_CASE c2 + // + // ... + // + // JSOP_JUMPTARGET + // JSOP_DEFAULT default + // + // c1: + // JSOP_JUMPTARGET + // {c1_body} + // JSOP_GOTO end + // + // c2: + // JSOP_JUMPTARGET + // {c2_body} + // JSOP_GOTO end + // + // default: + // end: + // JSOP_JUMPTARGET + // + // Table Switch + // {discriminant} + // JSOP_TABLESWITCH c1, c2, ... + // + // c1: + // JSOP_JUMPTARGET + // {c1_body} + // JSOP_GOTO end + // + // c2: + // JSOP_JUMPTARGET + // {c2_body} + // JSOP_GOTO end + // + // ... + // + // end: + // JSOP_JUMPTARGET + + public: + enum class Kind { + Table, + Cond + }; + + // Class for generating optimized table switch data. + class MOZ_STACK_CLASS TableGenerator + { + BytecodeEmitter* bce_; + + // Bit array for given numbers. + mozilla::Maybe<js::Vector<size_t, 128, SystemAllocPolicy>> intmap_; + + // The length of the intmap_. + int32_t intmapBitLength_ = 0; + + // The length of the table. + uint32_t tableLength_ = 0; + + // The lower and higher bounds of the table. + int32_t low_ = JSVAL_INT_MAX, high_ = JSVAL_INT_MIN; + + // Whether the table is still valid. + bool valid_= true; + +#ifdef DEBUG + bool finished_ = false; +#endif + + public: + explicit TableGenerator(BytecodeEmitter* bce) + : bce_(bce) + {} + + void setInvalid() { + valid_ = false; + } + MOZ_MUST_USE bool isValid() const { + return valid_; + } + MOZ_MUST_USE bool isInvalid() const { + return !valid_; + } + + // Add the given number to the table. The number is the value of + // `expr` for `case expr:` syntax. + MOZ_MUST_USE bool addNumber(int32_t caseValue); + + // Finish generating the table. + // `caseCount` should be the number of cases in the switch statement, + // excluding the default case. + void finish(uint32_t caseCount); + + private: + friend SwitchEmitter; + + // The following methods can be used only after calling `finish`. + + // Returns the lower bound of the added numbers. + int32_t low() const { + MOZ_ASSERT(finished_); + return low_; + } + + // Returns the higher bound of the numbers. + int32_t high() const { + MOZ_ASSERT(finished_); + return high_; + } + + // Returns the index in SwitchEmitter.caseOffsets_ for table switch. + uint32_t toCaseIndex(int32_t caseValue) const; + + // Returns the length of the table. + // This method can be called only if `isValid()` is true. + uint32_t tableLength() const; + }; + + private: + BytecodeEmitter* bce_; + + // `kind_` should be set to the correct value in emitCond/emitTable. + Kind kind_ = Kind::Cond; + + // True if there's explicit default case. + bool hasDefault_ = false; + + // The source note index for SRC_CONDSWITCH. + unsigned noteIndex_ = 0; + + // Source note index of the previous SRC_NEXTCASE. + unsigned caseNoteIndex_ = 0; + + // The number of cases in the switch statement, excluding the default case. + uint32_t caseCount_ = 0; + + // Internal index for case jump and case body, used by cond switch. + uint32_t caseIndex_ = 0; + + // Bytecode offset after emitting `discriminant`. + ptrdiff_t top_ = 0; + + // Bytecode offset of the previous JSOP_CASE. + ptrdiff_t lastCaseOffset_ = 0; + + // Bytecode offset of the JSOP_JUMPTARGET for default body. + JumpTarget defaultJumpTargetOffset_ = { -1 }; + + // Bytecode offset of the JSOP_DEFAULT. + JumpList condSwitchDefaultOffset_; + + // Instantiated when there's lexical scope for entire switch. + mozilla::Maybe<TDZCheckCache> tdzCacheLexical_; + mozilla::Maybe<EmitterScope> emitterScope_; + + // Instantiated while emitting case expression and case/default body. + mozilla::Maybe<TDZCheckCache> tdzCacheCaseAndBody_; + + // Control for switch. + mozilla::Maybe<BreakableControl> controlInfo_; + + mozilla::Maybe<uint32_t> switchPos_; + + // Cond Switch: + // Offset of each JSOP_CASE. + // Table Switch: + // Offset of each JSOP_JUMPTARGET for case. + js::Vector<ptrdiff_t, 32, SystemAllocPolicy> caseOffsets_; + + // The state of this emitter. + // + // +-------+ emitDiscriminant +--------------+ + // | Start |----------------->| Discriminant |-+ + // +-------+ +--------------+ | + // | + // +-------------------------------------------+ + // | + // | validateCaseCount +-----------+ + // +->+------------------------>+------------------>| CaseCount |-+ + // | ^ +-----------+ | + // | emitLexical +---------+ | | + // +------------>| Lexical |-+ | + // +---------+ | + // | + // +--------------------------------------------------------------+ + // | + // | emitTable +-------+ + // +---------->| Table |---------------------------->+-+ + // | +-------+ ^ | + // | | | + // | emitCond +------+ | | + // +---------->| Cond |-+------------------------>+->+ | + // +------+ | ^ | + // | | | + // | emitCase +------+ | | + // +->+--------->| Case |->+-+ | + // ^ +------+ | | + // | | | + // +--------------------+ | + // | + // +---------------------------------------------------+ + // | + // | emitEnd +-----+ + // +-+----------------------------------------->+-------->| End | + // | ^ +-----+ + // | emitCaseBody +----------+ | + // +->+-+---------------->| CaseBody |--->+-+-+ + // ^ | +----------+ ^ | + // | | | | + // | | emitDefaultBody +-------------+ | | + // | +---------------->| DefaultBody |-+ | + // | +-------------+ | + // | | + // +-------------------------------------+ + // + enum class State { + // The initial state. + Start, + + // After calling emitDiscriminant. + Discriminant, + + // After calling validateCaseCount. + CaseCount, + + // After calling emitLexical. + Lexical, + + // After calling emitCond. + Cond, + + // After calling emitTable. + Table, + + // After calling emitCase. + Case, + + // After calling emitCaseBody. + CaseBody, + + // After calling emitDefaultBody. + DefaultBody, + + // After calling emitEnd. + End + }; + State state_ = State::Start; + + public: + explicit SwitchEmitter(BytecodeEmitter* bce); + + // `switchPos` is the offset in the source code for the character below: + // + // switch ( cond ) { ... } + // ^ + // | + // switchPos + // + // Can be Nothing() if not available. + MOZ_MUST_USE bool emitDiscriminant(const mozilla::Maybe<uint32_t>& switchPos); + + // `caseCount` should be the number of cases in the switch statement, + // excluding the default case. + MOZ_MUST_USE bool validateCaseCount(uint32_t caseCount); + + // `bindings` is a lexical scope for the entire switch, in case there's + // let/const effectively directly under case or default blocks. + MOZ_MUST_USE bool emitLexical(Handle<LexicalScope::Data*> bindings); + + MOZ_MUST_USE bool emitCond(); + MOZ_MUST_USE bool emitTable(const TableGenerator& tableGen); + + MOZ_MUST_USE bool emitCaseJump(); + + MOZ_MUST_USE bool emitCaseBody(); + MOZ_MUST_USE bool emitCaseBody(int32_t caseValue, const TableGenerator& tableGen); + MOZ_MUST_USE bool emitDefaultBody(); + MOZ_MUST_USE bool emitEnd(); + + private: + MOZ_MUST_USE bool emitCaseOrDefaultJump(uint32_t caseIndex, bool isDefault); + MOZ_MUST_USE bool emitImplicitDefault(); +}; + +} /* namespace frontend */ +} /* namespace js */ + +#endif /* frontend_SwitchEmitter_h */
--- a/js/src/frontend/TokenStream.cpp +++ b/js/src/frontend/TokenStream.cpp @@ -435,16 +435,42 @@ TokenStreamAnyChars::TokenStreamAnyChars template<typename CharT> TokenStreamCharsBase<CharT>::TokenStreamCharsBase(JSContext* cx, const CharT* chars, size_t length, size_t startOffset) : TokenStreamCharsShared(cx), sourceUnits(chars, length, startOffset) {} +template<> +MOZ_MUST_USE bool +TokenStreamCharsBase<char16_t>::fillCharBufferWithTemplateStringContents(const char16_t* cur, + const char16_t* end) +{ + MOZ_ASSERT(this->charBuffer.length() == 0); + + while (cur < end) { + // Template literals normalize only '\r' and "\r\n" to '\n'. The + // Unicode separators need no special handling here. + // https://tc39.github.io/ecma262/#sec-static-semantics-tv-and-trv + char16_t ch = *cur++; + if (ch == '\r') { + ch = '\n'; + if (cur < end && *cur == '\n') + cur++; + } + + if (!this->charBuffer.append(ch)) + return false; + } + + MOZ_ASSERT(cur == end); + return true; +} + template<typename CharT, class AnyCharsAccess> TokenStreamSpecific<CharT, AnyCharsAccess>::TokenStreamSpecific(JSContext* cx, const ReadOnlyCompileOptions& options, const CharT* base, size_t length) : TokenStreamChars<CharT, AnyCharsAccess>(cx, base, length, options.scriptSourceOffset) {} bool @@ -482,19 +508,16 @@ void TokenStreamAnyChars::undoInternalUpdateLineInfoForEOL() { MOZ_ASSERT(prevLinebase != size_t(-1)); // we should never get more than one EOL linebase = prevLinebase; prevLinebase = size_t(-1); lineno--; } -// This gets a full code point, starting from an already-consumed leading code -// unit, normalizing EOL sequences to '\n', also updating line/column info as -// needed. template<class AnyCharsAccess> bool TokenStreamChars<char16_t, AnyCharsAccess>::getCodePoint(int32_t* cp) { TokenStreamAnyChars& anyChars = anyCharsAccess(); if (MOZ_UNLIKELY(this->sourceUnits.atEnd())) { anyChars.flags.isEOF = true; @@ -505,20 +528,17 @@ TokenStreamChars<char16_t, AnyCharsAcces int32_t c = this->sourceUnits.getCodeUnit(); do { // Normalize the char16_t if it was a newline. if (MOZ_UNLIKELY(c == '\n')) break; if (MOZ_UNLIKELY(c == '\r')) { - // If it's a \r\n sequence: treat as a single EOL, skip over the \n. - if (MOZ_LIKELY(!this->sourceUnits.atEnd())) - this->sourceUnits.matchCodeUnit('\n'); - + matchLineTerminator('\n'); break; } if (MOZ_UNLIKELY(c == unicode::LINE_SEPARATOR || c == unicode::PARA_SEPARATOR)) break; *cp = c; return true; @@ -596,31 +616,16 @@ TokenStreamChars<char16_t, AnyCharsAcces unicode::UTF16Encode(codePoint, units, &numUnits); MOZ_ASSERT(numUnits == 1 || numUnits == 2); while (numUnits-- > 0) ungetCodeUnit(units[numUnits]); } -template<class AnyCharsAccess> -void -TokenStreamChars<char16_t, AnyCharsAccess>::ungetLineTerminator() -{ - this->sourceUnits.ungetCodeUnit(); - - char16_t last = this->sourceUnits.peekCodeUnit(); - MOZ_ASSERT(SourceUnits::isRawEOLChar(last)); - - if (last == '\n') - this->sourceUnits.ungetOptionalCRBeforeLF(); - - anyCharsAccess().undoInternalUpdateLineInfoForEOL(); -} - template<typename CharT> size_t SourceUnits<CharT>::findEOLMax(size_t start, size_t max) { const CharT* p = codeUnitPtrAt(start); size_t n = 0; while (true) { @@ -1324,30 +1329,25 @@ TokenStreamSpecific<CharT, AnyCharsAcces return false; continue; } if (unit != '\\' || !matchUnicodeEscapeIdent(&codePoint)) break; } else { - int32_t cp; - if (!getNonAsciiCodePoint(unit, &cp)) + // |restoreNextRawCharAddress| undoes all gets, and this function + // doesn't update line/column info. + char32_t cp; + if (!getNonAsciiCodePointDontNormalize(unit, &cp)) return false; - codePoint = AssertedCast<uint32_t>(cp); - - if (!unicode::IsIdentifierPart(codePoint)) { - if (MOZ_UNLIKELY(codePoint == '\n')) { - // |restoreNextRawCharAddress| will undo all gets, but we - // have to revert a line/column update manually. - anyCharsAccess().undoInternalUpdateLineInfoForEOL(); - } + codePoint = cp; + if (!unicode::IsIdentifierPart(codePoint)) break; - } } if (!appendCodePointToCharBuffer(codePoint)) return false; } while (true); return true; } @@ -1506,26 +1506,26 @@ static const uint8_t firstCharKinds[] = #undef T_RB #undef T_LC #undef T_RC #undef _______ static_assert(LastCharKind < (1 << (sizeof(firstCharKinds[0]) * 8)), "Elements of firstCharKinds[] are too small"); -template<typename CharT, class AnyCharsAccess> void -GeneralTokenStreamChars<CharT, AnyCharsAccess>::consumeRestOfSingleLineComment() +SpecializedTokenStreamCharsBase<char16_t>::infallibleConsumeRestOfSingleLineComment() { - int32_t c; - do { - c = getCodeUnit(); - } while (c != EOF && !SourceUnits::isRawEOLChar(c)); - - ungetCodeUnit(c); + while (MOZ_LIKELY(!this->sourceUnits.atEnd())) { + char16_t unit = this->sourceUnits.peekCodeUnit(); + if (SourceUnits::isRawEOLChar(unit)) + return; + + this->sourceUnits.consumeKnownCodeUnit(unit); + } } template<typename CharT, class AnyCharsAccess> MOZ_MUST_USE bool TokenStreamSpecific<CharT, AnyCharsAccess>::decimalNumber(int32_t unit, TokenStart start, const CharT* numStart, Modifier modifier, TokenKind* out) { @@ -1623,23 +1623,26 @@ template<typename CharT, class AnyCharsA MOZ_MUST_USE bool TokenStreamSpecific<CharT, AnyCharsAccess>::regexpLiteral(TokenStart start, TokenKind* out) { MOZ_ASSERT(this->sourceUnits.previousCodeUnit() == '/'); this->charBuffer.clear(); auto ProcessNonAsciiCodePoint = [this](int32_t lead) { MOZ_ASSERT(lead != EOF); - - int32_t codePoint; - if (!this->getNonAsciiCodePoint(lead, &codePoint)) + MOZ_ASSERT(!this->isAsciiCodePoint(lead)); + + char32_t codePoint; + if (!this->getNonAsciiCodePointDontNormalize(lead, &codePoint)) return false; - if (codePoint == '\n') { - this->ungetLineTerminator(); + if (MOZ_UNLIKELY(codePoint == unicode::LINE_SEPARATOR || + codePoint == unicode::PARA_SEPARATOR)) + { + this->sourceUnits.ungetLineOrParagraphSeparator(); this->reportError(JSMSG_UNTERMINATED_REGEXP); return false; } return this->appendCodePointToCharBuffer(codePoint); }; auto ReportUnterminatedRegExp = [this](CharT unit) { @@ -1650,55 +1653,63 @@ TokenStreamSpecific<CharT, AnyCharsAcces bool inCharClass = false; do { int32_t unit = getCodeUnit(); if (unit == EOF) { ReportUnterminatedRegExp(unit); return badToken(); } - if (MOZ_LIKELY(isAsciiCodePoint(unit))) { - if (unit == '\\') { - if (!this->charBuffer.append(unit)) - return badToken(); - - unit = getCodeUnit(); - if (unit == EOF) { - ReportUnterminatedRegExp(unit); - return badToken(); - } - - // Fallthrough only handles ASCII code points, so - // deal with non-ASCII and skip everything else. - if (MOZ_UNLIKELY(!isAsciiCodePoint(unit))) { - if (!ProcessNonAsciiCodePoint(unit)) - return badToken(); - - continue; - } - } else if (unit == '[') { - inCharClass = true; - } else if (unit == ']') { - inCharClass = false; - } else if (unit == '/' && !inCharClass) { - // For IE compat, allow unescaped / in char classes. - break; - } - - if (unit == '\r' || unit == '\n') { + if (MOZ_UNLIKELY(!isAsciiCodePoint(unit))) { + if (!ProcessNonAsciiCodePoint(unit)) + return badToken(); + + continue; + } + + if (unit == '\\') { + if (!this->charBuffer.append(unit)) + return badToken(); + + unit = getCodeUnit(); + if (unit == EOF) { ReportUnterminatedRegExp(unit); return badToken(); } - if (!this->charBuffer.append(unit)) - return badToken(); - } else { - if (!ProcessNonAsciiCodePoint(unit)) - return badToken(); + // Fallthrough only handles ASCII code points, so + // deal with non-ASCII and skip everything else. + if (MOZ_UNLIKELY(!isAsciiCodePoint(unit))) { + if (!ProcessNonAsciiCodePoint(unit)) + return badToken(); + + continue; + } + } else if (unit == '[') { + inCharClass = true; + } else if (unit == ']') { + inCharClass = false; + } else if (unit == '/' && !inCharClass) { + // For IE compat, allow unescaped / in char classes. + break; } + + if (unit == '\r' || unit == '\n') { + ReportUnterminatedRegExp(unit); + return badToken(); + } + + // We're accumulating regular expression *source* text here: source + // text matching a line break will appear as U+005C REVERSE SOLIDUS + // U+006E LATIN SMALL LETTER N, and |unit| here would be the latter + // code point. + MOZ_ASSERT(!SourceUnits::isRawEOLChar(unit)); + + if (!this->charBuffer.append(unit)) + return badToken(); } while (true); int32_t unit; RegExpFlag reflags = NoFlags; while (true) { RegExpFlag flag; unit = getCodeUnit(); if (unit == 'g') @@ -1850,19 +1861,18 @@ TokenStreamSpecific<CharT, AnyCharsAcces // Look for a string or a template string. // if (c1kind == String) return getStringOrTemplateToken(static_cast<char>(unit), modifier, ttp); // Skip over EOL chars, updating line state along the way. // if (c1kind == EOL) { - // If it's a \r\n sequence, consume it as a single EOL. - if (unit == '\r' && !this->sourceUnits.atEnd()) - this->sourceUnits.matchCodeUnit('\n'); + if (unit == '\r') + matchLineTerminator('\n'); if (!updateLineInfoForEOL()) return badToken(); anyCharsAccess().updateFlagsForEOL(); continue; } @@ -2092,17 +2102,19 @@ TokenStreamSpecific<CharT, AnyCharsAcces break; case '<': if (anyCharsAccess().options().allowHTMLComments) { // Treat HTML begin-comment as comment-till-end-of-line. if (matchCodeUnit('!')) { if (matchCodeUnit('-')) { if (matchCodeUnit('-')) { - consumeRestOfSingleLineComment(); + if (!consumeRestOfSingleLineComment()) + return false; + continue; } ungetCodeUnit('-'); } ungetCodeUnit('!'); } } if (matchCodeUnit('<')) @@ -2137,17 +2149,19 @@ TokenStreamSpecific<CharT, AnyCharsAcces bool shouldWarn = unit == '@'; if (!getDirectives(false, shouldWarn)) return false; } else { // NOTE: |unit| may be EOF here. ungetCodeUnit(unit); } - consumeRestOfSingleLineComment(); + if (!consumeRestOfSingleLineComment()) + return false; + continue; } // Look for a multi-line comment. if (matchCodeUnit('*')) { TokenStreamAnyChars& anyChars = anyCharsAccess(); unsigned linenoBefore = anyChars.lineno; @@ -2194,17 +2208,19 @@ TokenStreamSpecific<CharT, AnyCharsAcces break; case '-': if (matchCodeUnit('-')) { if (anyCharsAccess().options().allowHTMLComments && !anyCharsAccess().flags.isDirtyLine) { if (matchCodeUnit('>')) { - consumeRestOfSingleLineComment(); + if (!consumeRestOfSingleLineComment()) + return false; + continue; } } simpleKind = TokenKind::Dec; } else { simpleKind = matchCodeUnit('=') ? TokenKind::SubAssign : TokenKind::Sub; } @@ -2337,18 +2353,17 @@ TokenStreamSpecific<CharT, AnyCharsAcces case 'b': unit = '\b'; break; case 'f': unit = '\f'; break; case 'n': unit = '\n'; break; case 'r': unit = '\r'; break; case 't': unit = '\t'; break; case 'v': unit = '\v'; break; case '\r': - if (MOZ_LIKELY(!this->sourceUnits.atEnd())) - this->sourceUnits.matchCodeUnit('\n'); + matchLineTerminator('\n'); MOZ_FALLTHROUGH; case '\n': { // LineContinuation represents no code points. We're manually // consuming a LineTerminatorSequence, so we must manually // update line/column info. if (!updateLineInfoForEOL()) return false; @@ -2544,20 +2559,17 @@ TokenStreamSpecific<CharT, AnyCharsAcces // String literals don't allow ASCII line breaks. ungetCodeUnit(unit); ReportPrematureEndOfLiteral(JSMSG_EOL_BEFORE_END_OF_STRING); return false; } if (unit == '\r') { unit = '\n'; - - // If it's a \r\n sequence: treat as a single EOL, skip over the \n. - if (!this->sourceUnits.atEnd()) - this->sourceUnits.matchCodeUnit('\n'); + matchLineTerminator('\n'); } if (!updateLineInfoForEOL()) return false; anyCharsAccess().updateFlagsForEOL(); } else if (parsingTemplate && unit == '$' && matchCodeUnit('{')) { templateHead = true; @@ -2610,31 +2622,31 @@ TokenKindToString(TokenKind tt) #undef EMIT_CASE case TokenKind::Limit: break; } return "<bad TokenKind>"; } #endif -template class frontend::TokenStreamCharsBase<Utf8Unit>; -template class frontend::TokenStreamCharsBase<char16_t>; - -template class frontend::TokenStreamChars<char16_t, frontend::TokenStreamAnyCharsAccess>; -template class frontend::TokenStreamSpecific<char16_t, frontend::TokenStreamAnyCharsAccess>; +template class TokenStreamCharsBase<Utf8Unit>; +template class TokenStreamCharsBase<char16_t>; + +template class TokenStreamChars<char16_t, TokenStreamAnyCharsAccess>; +template class TokenStreamSpecific<char16_t, TokenStreamAnyCharsAccess>; template class -frontend::TokenStreamChars<char16_t, frontend::ParserAnyCharsAccess<frontend::GeneralParser<frontend::FullParseHandler, char16_t>>>; +TokenStreamChars<char16_t, ParserAnyCharsAccess<GeneralParser<FullParseHandler, char16_t>>>; template class -frontend::TokenStreamChars<char16_t, frontend::ParserAnyCharsAccess<frontend::GeneralParser<frontend::SyntaxParseHandler, char16_t>>>; +TokenStreamChars<char16_t, ParserAnyCharsAccess<GeneralParser<SyntaxParseHandler, char16_t>>>; template class -frontend::TokenStreamSpecific<char16_t, frontend::ParserAnyCharsAccess<frontend::GeneralParser<frontend::FullParseHandler, char16_t>>>; +TokenStreamSpecific<char16_t, ParserAnyCharsAccess<GeneralParser<FullParseHandler, char16_t>>>; template class -frontend::TokenStreamSpecific<char16_t, frontend::ParserAnyCharsAccess<frontend::GeneralParser<frontend::SyntaxParseHandler, char16_t>>>; +TokenStreamSpecific<char16_t, ParserAnyCharsAccess<GeneralParser<SyntaxParseHandler, char16_t>>>; } // namespace frontend } // namespace js JS_FRIEND_API(int) js_fgets(char* buf, int size, FILE* file)
--- a/js/src/frontend/TokenStream.h +++ b/js/src/frontend/TokenStream.h @@ -55,33 +55,51 @@ * potentially the compiler *can't* unify them because offsets into the * hypothetical TokenStream<CharT>s would differ.) Second, some of this stuff * needs to be accessible in ParserBase, the aspects of JS language parsing * that have meaning independent of the character type of the source text being * parsed. So we need a separate data structure that ParserBase can hold on to * for it. (ParserBase isn't the only instance of this, but it's certainly the * biggest case of it.) Ergo, TokenStreamAnyChars. * - * == TokenStreamCharsBase<CharT> → ∅ == + * == TokenStreamCharsShared → ∅ == + * + * Some functionality has meaning independent of character type, yet has no use + * *unless* you know the character type in actual use. It *could* live in + * TokenStreamAnyChars, but it makes more sense to live in a separate class + * that character-aware token information can simply inherit. * - * Certain data structures in tokenizing are character-type-specific: + * This class currently exists only to contain a char16_t buffer, transiently + * used to accumulate strings in tricky cases that can't just be read directly + * from source text. It's not used outside character-aware tokenizing, so it + * doesn't make sense in TokenStreamAnyChars. + * + * == TokenStreamCharsBase<CharT> → TokenStreamCharsShared == + * + * Certain data structures in tokenizing are character-type-specific: namely, * the various pointers identifying the source text (including current offset - * and end) , and the temporary vector into which characters are read/written - * in certain cases (think writing out the actual codepoints identified by an - * identifier containing a Unicode escape, to create the atom for the - * identifier: |a\u0062c| versus |abc|, for example). + * and end). * * Additionally, some functions operating on this data are defined the same way - * no matter what character type you have -- the offset being |offset - start| - * no matter whether those two variables are single- or double-byte pointers. + * no matter what character type you have (e.g. current offset in code units + * into the source text) or share a common interface regardless of character + * type (e.g. consume the next code unit if it has a given value). * * All such functionality lives in TokenStreamCharsBase<CharT>. * + * == SpecializedTokenStreamCharsBase<CharT> → TokenStreamCharsBase<CharT> == + * + * Certain tokenizing functionality is specific to a single character type. + * For example, JS's UTF-16 encoding recognizes no coding errors, because lone + * surrogates are not an error; but a UTF-8 encoding must recognize a variety + * of validation errors. Such functionality is defined only in the appropriate + * SpecializedTokenStreamCharsBase specialization. + * * == GeneralTokenStreamChars<CharT, AnyCharsAccess> → - * TokenStreamCharsBase<CharT> == + * SpecializedTokenStreamCharsBase<CharT> == * * Some functionality operates differently on different character types, just * as for TokenStreamCharsBase, but additionally requires access to character- * type-agnostic information in TokenStreamAnyChars. For example, getting the * next character performs different steps for different character types and * must access TokenStreamAnyChars to update line break information. * * Such functionality, if it can be defined using the same algorithm for all @@ -923,16 +941,19 @@ CodeUnitValue(char16_t unit) } constexpr uint8_t CodeUnitValue(mozilla::Utf8Unit unit) { return unit.toUint8(); } +template<typename CharT> +class TokenStreamCharsBase; + // This is the low-level interface to the JS source code buffer. It just gets // raw Unicode code units -- 16-bit char16_t units of source text that are not // (always) full code points, and 8-bit units of UTF-8 source text soon. // TokenStreams functions are layered on top and do some extra stuff like // converting all EOL sequences to '\n', tracking the line number, and setting // |flags.isEOF|. (The "raw" in "raw Unicode code units" refers to the lack of // EOL sequence normalization.) // @@ -1045,39 +1066,53 @@ class SourceUnits void unskipCodeUnits(uint32_t n) { MOZ_ASSERT(ptr, "shouldn't use poisoned SourceUnits"); MOZ_ASSERT(n <= mozilla::PointerRangeSize(base_, ptr), "shouldn't unskip beyond start of SourceUnits"); ptr -= n; } - bool matchCodeUnit(CharT c) { - if (*ptr == c) { // this will nullptr-crash if poisoned + private: + friend class TokenStreamCharsBase<CharT>; + + bool internalMatchCodeUnit(CharT c) { + MOZ_ASSERT(ptr, "shouldn't use poisoned SourceUnits"); + if (MOZ_LIKELY(!atEnd()) && *ptr == c) { ptr++; return true; } return false; } + public: + void consumeKnownCodeUnit(CharT c) { + MOZ_ASSERT(ptr, "shouldn't use poisoned SourceUnits"); + MOZ_ASSERT(*ptr == c, "consuming the wrong code unit"); + ptr++; + } + /** * Unget the '\n' (CR) that precedes a '\n' (LF), when ungetting a line * terminator that's a full "\r\n" sequence. If the prior code unit isn't * '\r', do nothing. */ void ungetOptionalCRBeforeLF() { MOZ_ASSERT(ptr, "shouldn't unget a '\\r' from poisoned SourceUnits"); MOZ_ASSERT(*ptr == CharT('\n'), "function should only be called when a '\\n' was just " "ungotten, and any '\\r' preceding it must also be " "ungotten"); if (*(ptr - 1) == CharT('\r')) ptr--; } + /** Unget U+2028 LINE SEPARATOR or U+2029 PARAGRAPH SEPARATOR. */ + inline void ungetLineOrParagraphSeparator(); + void ungetCodeUnit() { MOZ_ASSERT(!atStart(), "can't unget if currently at start"); MOZ_ASSERT(ptr); // make sure it hasn't been poisoned ptr--; } const CharT* addressOfNextCodeUnit(bool allowPoisoned = false) const { MOZ_ASSERT_IF(!allowPoisoned, ptr); // make sure it hasn't been poisoned @@ -1117,16 +1152,43 @@ class SourceUnits /** Limit for quick bounds check. */ const CharT* limit_; /** Next char to get. */ const CharT* ptr; }; +template<> +inline void +SourceUnits<char16_t>::ungetLineOrParagraphSeparator() +{ +#ifdef DEBUG + char16_t prev = previousCodeUnit(); +#endif + MOZ_ASSERT(prev == unicode::LINE_SEPARATOR || prev == unicode::PARA_SEPARATOR); + + ungetCodeUnit(); +} + +template<> +inline void +SourceUnits<mozilla::Utf8Unit>::ungetLineOrParagraphSeparator() +{ + unskipCodeUnits(3); + + MOZ_ASSERT(ptr[0].toUint8() == 0xE2); + MOZ_ASSERT(ptr[1].toUint8() == 0x80); + +#ifdef DEBUG + uint8_t last = ptr[2].toUint8(); +#endif + MOZ_ASSERT(last == 0xA8 || last == 0xA9); +} + class TokenStreamCharsShared { // Using char16_t (not CharT) is a simplifying decision that hopefully // eliminates the need for a UTF-8 regular expression parser and makes // |copyCharBufferTo| markedly simpler. using CharBuffer = Vector<char16_t, 32>; protected: @@ -1166,58 +1228,79 @@ class TokenStreamCharsShared CharBuffer& getCharBuffer() { return charBuffer; } }; template<typename CharT> class TokenStreamCharsBase : public TokenStreamCharsShared { protected: + TokenStreamCharsBase(JSContext* cx, const CharT* chars, size_t length, size_t startOffset); + /** * Convert a non-EOF code unit returned by |getCodeUnit()| or * |peekCodeUnit()| to a CharT code unit. */ inline CharT toCharT(int32_t codeUnitValue); void ungetCodeUnit(int32_t c) { if (c == EOF) return; sourceUnits.ungetCodeUnit(); } - public: - TokenStreamCharsBase(JSContext* cx, const CharT* chars, size_t length, size_t startOffset); - static MOZ_ALWAYS_INLINE JSAtom* atomizeSourceChars(JSContext* cx, const CharT* chars, size_t length); using SourceUnits = frontend::SourceUnits<CharT>; - /** Match a non-EOL, non-EOF code unit; return true iff it was matched. */ - inline bool matchCodeUnit(int32_t expect); + /** + * Try to match a non-LineTerminator ASCII code point. Return true iff it + * was matched. + */ + bool matchCodeUnit(char expect) { + MOZ_ASSERT(mozilla::IsAscii(expect)); + MOZ_ASSERT(expect != '\r'); + MOZ_ASSERT(expect != '\n'); + return this->sourceUnits.internalMatchCodeUnit(CharT(expect)); + } - protected: + /** + * Try to match an ASCII LineTerminator code point. Return true iff it was + * matched. + */ + bool matchLineTerminator(char expect) { + MOZ_ASSERT(expect == '\r' || expect == '\n'); + return this->sourceUnits.internalMatchCodeUnit(CharT(expect)); + } + + template<typename T> bool matchCodeUnit(T) = delete; + template<typename T> bool matchLineTerminator(T) = delete; + int32_t peekCodeUnit() { return MOZ_LIKELY(!sourceUnits.atEnd()) ? CodeUnitValue(sourceUnits.peekCodeUnit()) : EOF; } - void consumeKnownCodeUnit(int32_t unit) { - MOZ_ASSERT(unit != EOF, "shouldn't be matching EOF"); - MOZ_ASSERT(!sourceUnits.atEnd(), "must have units to consume"); -#ifdef DEBUG - CharT next = -#endif - sourceUnits.getCodeUnit(); - MOZ_ASSERT(CodeUnitValue(next) == unit, - "must be consuming the correct unit"); - } + /** Consume a known, non-EOF code unit. */ + inline void consumeKnownCodeUnit(int32_t unit); - MOZ_MUST_USE inline bool - fillCharBufferWithTemplateStringContents(const CharT* cur, const CharT* end); + // Forbid accidental calls to consumeKnownCodeUnit *not* with the single + // unit-or-EOF type. CharT should use SourceUnits::consumeKnownCodeUnit; + // CodeUnitValue() results should go through toCharT(), or better yet just + // use the original CharT. + template<typename T> inline void consumeKnownCodeUnit(T) = delete; + + /** + * Accumulate the provided range of already-validated (i.e. valid UTF-8, or + * anything if CharT is char16_t because JS permits lone and mispaired + * surrogates) raw template literal text (i.e. containing no escapes or + * substitutions) into |charBuffer|. + */ + MOZ_MUST_USE bool fillCharBufferWithTemplateStringContents(const CharT* cur, const CharT* end); protected: /** Code units in the source code being tokenized. */ SourceUnits sourceUnits; }; template<> inline char16_t @@ -1230,59 +1313,103 @@ TokenStreamCharsBase<char16_t>::toCharT( template<> inline mozilla::Utf8Unit TokenStreamCharsBase<mozilla::Utf8Unit>::toCharT(int32_t value) { MOZ_ASSERT(value != EOF, "EOF is not a CharT"); return mozilla::Utf8Unit(static_cast<unsigned char>(value)); } +template<typename CharT> +inline void +TokenStreamCharsBase<CharT>::consumeKnownCodeUnit(int32_t unit) +{ + sourceUnits.consumeKnownCodeUnit(toCharT(unit)); +} + template<> /* static */ MOZ_ALWAYS_INLINE JSAtom* TokenStreamCharsBase<char16_t>::atomizeSourceChars(JSContext* cx, const char16_t* chars, size_t length) { return AtomizeChars(cx, chars, length); } template<typename CharT> -inline bool -TokenStreamCharsBase<CharT>::matchCodeUnit(int32_t expect) -{ - MOZ_ASSERT(expect != EOF, "shouldn't be matching EOFs"); - MOZ_ASSERT(!SourceUnits::isRawEOLChar(expect)); - return MOZ_LIKELY(!this->sourceUnits.atEnd()) && - this->sourceUnits.matchCodeUnit(toCharT(expect)); -} +class SpecializedTokenStreamCharsBase; template<> -MOZ_MUST_USE inline bool -TokenStreamCharsBase<char16_t>::fillCharBufferWithTemplateStringContents(const char16_t* cur, - const char16_t* end) +class SpecializedTokenStreamCharsBase<char16_t> + : public TokenStreamCharsBase<char16_t> { - MOZ_ASSERT(this->charBuffer.length() == 0); + using CharsBase = TokenStreamCharsBase<char16_t>; + + protected: + using TokenStreamCharsShared::isAsciiCodePoint; + // Deliberately don't |using| |sourceUnits| because of bug 1472569. :-( + + using typename CharsBase::SourceUnits; + + protected: + // These APIs are only usable by UTF-16-specific code. - while (cur < end) { - // U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR are - // interpreted literally inside template literal contents; only - // literal CRLF sequences are normalized to '\n'. See - // <https://tc39.github.io/ecma262/#sec-static-semantics-tv-and-trv>. - char16_t ch = *cur++; - if (ch == '\r') { - ch = '\n'; - if (cur < end && *cur == '\n') - cur++; + /** + * Consume the rest of a single-line comment (but not the EOL/EOF that + * terminates it) -- infallibly because no 16-bit code unit sequence in a + * comment is an error. + */ + void infallibleConsumeRestOfSingleLineComment(); + + /** + * Given |lead| already consumed, consume and return the code point encoded + * starting from it. Infallible because lone surrogates in JS encode a + * "code point" of the same value. + */ + char32_t infallibleGetNonAsciiCodePointDontNormalize(char16_t lead) { + MOZ_ASSERT(!isAsciiCodePoint(lead)); + MOZ_ASSERT(this->sourceUnits.previousCodeUnit() == lead); + + // Handle single-unit code points and lone trailing surrogates. + if (MOZ_LIKELY(!unicode::IsLeadSurrogate(lead)) || + // Or handle lead surrogates not paired with trailing surrogates. + MOZ_UNLIKELY(this->sourceUnits.atEnd() || + !unicode::IsTrailSurrogate(this->sourceUnits.peekCodeUnit()))) + { + return lead; } - if (!this->charBuffer.append(ch)) - return false; + // Otherwise it's a multi-unit code point. + return unicode::UTF16Decode(lead, this->sourceUnits.getCodeUnit()); } - return true; -} + protected: + // These APIs are in both SpecializedTokenStreamCharsBase specializations + // and so are usable in subclasses no matter what CharT is. + + using CharsBase::CharsBase; +}; + +template<> +class SpecializedTokenStreamCharsBase<mozilla::Utf8Unit> + : public TokenStreamCharsBase<mozilla::Utf8Unit> +{ + using CharsBase = TokenStreamCharsBase<mozilla::Utf8Unit>; + + protected: + // Deliberately don't |using| |sourceUnits| because of bug 1472569. :-( + + protected: + // These APIs are only usable by UTF-8-specific code. + + protected: + // These APIs are in both SpecializedTokenStreamCharsBase specializations + // and so are usable in subclasses no matter what CharT is. + + using CharsBase::CharsBase; +}; /** A small class encapsulating computation of the start-offset of a Token. */ class TokenStart { uint32_t startOffset_; public: /** @@ -1297,19 +1424,20 @@ class TokenStart TokenStart(const TokenStart&) = default; uint32_t offset() const { return startOffset_; } }; template<typename CharT, class AnyCharsAccess> class GeneralTokenStreamChars - : public TokenStreamCharsBase<CharT> + : public SpecializedTokenStreamCharsBase<CharT> { using CharsBase = TokenStreamCharsBase<CharT>; + using SpecializedCharsBase = SpecializedTokenStreamCharsBase<CharT>; Token* newTokenInternal(TokenKind kind, TokenStart start, TokenKind* out); /** * Allocates a new Token from the given offset to the current offset, * ascribes it the given kind, and sets |*out| to that kind. */ Token* newToken(TokenKind kind, TokenStart start, TokenStreamShared::Modifier modifier, @@ -1327,20 +1455,23 @@ class GeneralTokenStreamChars return token; } uint32_t matchUnicodeEscape(uint32_t* codePoint); uint32_t matchExtendedUnicodeEscape(uint32_t* codePoint); protected: + using TokenStreamCharsShared::drainCharBufferIntoAtom; + using CharsBase::fillCharBufferWithTemplateStringContents; + using typename CharsBase::SourceUnits; protected: - using CharsBase::CharsBase; + using SpecializedCharsBase::SpecializedCharsBase; TokenStreamAnyChars& anyCharsAccess() { return AnyCharsAccess::anyChars(this); } const TokenStreamAnyChars& anyCharsAccess() const { return AnyCharsAccess::anyChars(this); } @@ -1412,62 +1543,96 @@ class GeneralTokenStreamChars } void ungetCodeUnit(int32_t c) { MOZ_ASSERT_IF(c == EOF, anyCharsAccess().flags.isEOF); CharsBase::ungetCodeUnit(c); } - /** - * Consume code units til EOL/EOF following the start of a single-line - * comment, without consuming the EOL/EOF. - */ - void consumeRestOfSingleLineComment(); - MOZ_MUST_USE MOZ_ALWAYS_INLINE bool updateLineInfoForEOL() { return anyCharsAccess().internalUpdateLineInfoForEOL(this->sourceUnits.offset()); } uint32_t matchUnicodeEscapeIdStart(uint32_t* codePoint); bool matchUnicodeEscapeIdent(uint32_t* codePoint); + + public: + JSAtom* getRawTemplateStringAtom() { + TokenStreamAnyChars& anyChars = anyCharsAccess(); + + MOZ_ASSERT(anyChars.currentToken().type == TokenKind::TemplateHead || + anyChars.currentToken().type == TokenKind::NoSubsTemplate); + const CharT* cur = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.begin + 1); + const CharT* end; + if (anyChars.currentToken().type == TokenKind::TemplateHead) { + // Of the form |`...${| or |}...${| + end = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 2); + } else { + // NO_SUBS_TEMPLATE is of the form |`...`| or |}...`| + end = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 1); + } + + if (!fillCharBufferWithTemplateStringContents(cur, end)) + return nullptr; + + return drainCharBufferIntoAtom(anyChars.cx); + } }; template<typename CharT, class AnyCharsAccess> class TokenStreamChars; template<class AnyCharsAccess> class TokenStreamChars<char16_t, AnyCharsAccess> : public GeneralTokenStreamChars<char16_t, AnyCharsAccess> { private: using CharsBase = TokenStreamCharsBase<char16_t>; + using SpecializedCharsBase = SpecializedTokenStreamCharsBase<char16_t>; using GeneralCharsBase = GeneralTokenStreamChars<char16_t, AnyCharsAccess>; using Self = TokenStreamChars<char16_t, AnyCharsAccess>; using GeneralCharsBase::asSpecific; using typename GeneralCharsBase::TokenStreamSpecific; protected: using GeneralCharsBase::anyCharsAccess; using GeneralCharsBase::getCodeUnit; + using SpecializedCharsBase::infallibleConsumeRestOfSingleLineComment; + using SpecializedCharsBase::infallibleGetNonAsciiCodePointDontNormalize; using TokenStreamCharsShared::isAsciiCodePoint; + using CharsBase::matchLineTerminator; // Deliberately don't |using| |sourceUnits| because of bug 1472569. :-( using GeneralCharsBase::ungetCodeUnit; using GeneralCharsBase::updateLineInfoForEOL; using typename GeneralCharsBase::SourceUnits; protected: using GeneralCharsBase::GeneralCharsBase; - // Try to get the next code point, normalizing '\r', '\r\n', '\n', and the - // Unicode line/paragraph separators into '\n'. Also updates internal - // line-counter state. Return true on success and store the code point in - // |*c|. Return false and leave |*c| undefined on failure. + /** + * Given the non-ASCII |lead| code unit just consumed, consume and return a + * complete non-ASCII code point. Line/column updates are not performed, + * and line breaks are returned as-is without normalization. + */ + MOZ_MUST_USE bool getNonAsciiCodePointDontNormalize(char16_t lead, char32_t* codePoint) { + // There are no encoding errors in 16-bit JS, so implement this so that + // the compiler knows it, too. + *codePoint = infallibleGetNonAsciiCodePointDontNormalize(lead); + return true; + } + + /** + * Get the next code point, converting LineTerminatorSequences to '\n' and + * updating internal line-counter state if needed. Return true on success + * and store the code point in |*c|. Return false and leave |*c| undefined + * on failure. + */ MOZ_MUST_USE bool getCodePoint(int32_t* cp); /** * Given a just-consumed ASCII code unit/point |lead|, consume a full code * point or LineTerminatorSequence (normalizing it to '\n') and store it in * |*codePoint|. Return true on success, otherwise return false and leave * |*codePoint| undefined on failure. * @@ -1478,19 +1643,17 @@ class TokenStreamChars<char16_t, AnyChar MOZ_MUST_USE bool getFullAsciiCodePoint(int32_t lead, int32_t* codePoint) { MOZ_ASSERT(isAsciiCodePoint(lead), "non-ASCII code units must be handled separately"); // NOTE: |this->|-qualify to avoid a gcc bug: see bug 1472569. MOZ_ASSERT(lead == this->sourceUnits.previousCodeUnit(), "getFullAsciiCodePoint called incorrectly"); if (MOZ_UNLIKELY(lead == '\r')) { - // NOTE: |this->|-qualify to avoid a gcc bug: see bug 1472569. - if (MOZ_LIKELY(!this->sourceUnits.atEnd())) - this->sourceUnits.matchCodeUnit('\n'); + matchLineTerminator('\n'); } else if (MOZ_LIKELY(lead != '\n')) { *codePoint = lead; return true; } *codePoint = '\n'; bool ok = updateLineInfoForEOL(); if (!ok) { @@ -1498,26 +1661,27 @@ class TokenStreamChars<char16_t, AnyChar *codePoint = EOF; // sentinel value to hopefully cause errors #endif MOZ_MAKE_MEM_UNDEFINED(codePoint, sizeof(*codePoint)); } return ok; } /** - * Given a just-consumed non-ASCII code unit (and maybe point) |lead|, - * consume a full code point or LineTerminatorSequence (normalizing it to - * '\n') and store it in |*codePoint|. Return true on success, otherwise - * return false and leave |*codePoint| undefined on failure. + * Given a just-consumed non-ASCII code unit |lead| (which may also be a + * full code point, for UTF-16), consume a full code point or + * LineTerminatorSequence (normalizing it to '\n') and store it in + * |*codePoint|. Return true on success, otherwise return false and leave + * |*codePoint| undefined on failure. * * If a LineTerminatorSequence was consumed, also update line/column info. * * This may change the current |sourceUnits| offset. */ - MOZ_MUST_USE bool getNonAsciiCodePoint(int32_t lead, int32_t* cp); + MOZ_MUST_USE bool getNonAsciiCodePoint(int32_t lead, int32_t* codePoint); /** * Unget a full code point (ASCII or not) without altering line/column * state. If line/column state must be updated, this must happen manually. * This method ungets a single code point, not a LineTerminatorSequence * that is multiple code points. (Generally you shouldn't be in a state * where you've just consumed "\r\n" and want to unget that full sequence.) * @@ -1540,21 +1704,25 @@ class TokenStreamChars<char16_t, AnyChar "should not be ungetting un-normalized code points"); ungetCodePointIgnoreEOL(codePoint); if (codePoint == '\n') anyCharsAccess().undoInternalUpdateLineInfoForEOL(); } /** - * Unget a just-gotten LineTerminator sequence: '\r', '\n', '\r\n', or - * a Unicode line/paragraph separator, also undoing line/column information - * changes reflecting that LineTerminator. + * Consume code points til EOL/EOF following the start of a single-line + * comment, without consuming the EOL/EOF. */ - void ungetLineTerminator(); + MOZ_MUST_USE bool consumeRestOfSingleLineComment() { + // This operation is infallible for UTF-16 -- and this implementation + // approach lets the compiler boil away call-side fallibility handling. + infallibleConsumeRestOfSingleLineComment(); + return true; + } }; // TokenStream is the lexical scanner for JavaScript source text. // // It takes a buffer of CharT code units (currently only char16_t encoding // UTF-16, but we're adding either UTF-8 or Latin-1 single-byte text soon) and // linearly scans it into |Token|s. // @@ -1596,18 +1764,19 @@ class TokenStreamChars<char16_t, AnyChar template<typename CharT, class AnyCharsAccess> class MOZ_STACK_CLASS TokenStreamSpecific : public TokenStreamChars<CharT, AnyCharsAccess>, public TokenStreamShared, public ErrorReporter { public: using CharsBase = TokenStreamCharsBase<CharT>; + using SpecializedCharsBase = SpecializedTokenStreamCharsBase<CharT>; using GeneralCharsBase = GeneralTokenStreamChars<CharT, AnyCharsAccess>; - using SpecializedCharsBase = TokenStreamChars<CharT, AnyCharsAccess>; + using SpecializedChars = TokenStreamChars<CharT, AnyCharsAccess>; using Position = TokenStreamPosition<CharT>; // Anything inherited through a base class whose type depends upon this // class's template parameters can only be accessed through a dependent // name: prefixed with |this|, by explicit qualification, and so on. (This // is so that references to inherited fields are statically distinguishable // from references to names outside of the class.) This is tedious and @@ -1624,38 +1793,40 @@ class MOZ_STACK_CLASS TokenStreamSpecifi using typename CharsBase::SourceUnits; private: using TokenStreamCharsShared::appendCodePointToCharBuffer; using CharsBase::atomizeSourceChars; using GeneralCharsBase::badToken; // Deliberately don't |using| |charBuffer| because of bug 1472569. :-( using CharsBase::consumeKnownCodeUnit; - using GeneralCharsBase::consumeRestOfSingleLineComment; + using SpecializedChars::consumeRestOfSingleLineComment; using TokenStreamCharsShared::copyCharBufferTo; using TokenStreamCharsShared::drainCharBufferIntoAtom; using CharsBase::fillCharBufferWithTemplateStringContents; - using SpecializedCharsBase::getCodePoint; + using SpecializedChars::getCodePoint; using GeneralCharsBase::getCodeUnit; - using SpecializedCharsBase::getFullAsciiCodePoint; - using SpecializedCharsBase::getNonAsciiCodePoint; + using SpecializedChars::getFullAsciiCodePoint; + using SpecializedChars::getNonAsciiCodePoint; + using SpecializedChars::getNonAsciiCodePointDontNormalize; using TokenStreamCharsShared::isAsciiCodePoint; using CharsBase::matchCodeUnit; + using CharsBase::matchLineTerminator; using GeneralCharsBase::matchUnicodeEscapeIdent; using GeneralCharsBase::matchUnicodeEscapeIdStart; using GeneralCharsBase::newAtomToken; using GeneralCharsBase::newNameToken; using GeneralCharsBase::newNumberToken; using GeneralCharsBase::newRegExpToken; using GeneralCharsBase::newSimpleToken; using CharsBase::peekCodeUnit; // Deliberately don't |using| |sourceUnits| because of bug 1472569. :-( - using SpecializedCharsBase::ungetCodePointIgnoreEOL; + using SpecializedChars::ungetCodePointIgnoreEOL; using GeneralCharsBase::ungetCodeUnit; - using SpecializedCharsBase::ungetNonAsciiNormalizedCodePoint; + using SpecializedChars::ungetNonAsciiNormalizedCodePoint; using GeneralCharsBase::updateLineInfoForEOL; template<typename CharU> friend class TokenStreamPosition; public: TokenStreamSpecific(JSContext* cx, const ReadOnlyCompileOptions& options, const CharT* base, size_t length); @@ -1733,37 +1904,16 @@ class MOZ_STACK_CLASS TokenStreamSpecifi // These functions take a |va_list*| parameter, not a |va_list| parameter, // to hack around bug 1363116. (Longer-term, the right fix is of course to // not use ellipsis functions or |va_list| at all in error reporting.) bool reportStrictModeErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset, bool strictMode, unsigned errorNumber, va_list* args); bool reportExtraWarningErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset, unsigned errorNumber, va_list* args); - JSAtom* getRawTemplateStringAtom() { - TokenStreamAnyChars& anyChars = anyCharsAccess(); - - MOZ_ASSERT(anyChars.currentToken().type == TokenKind::TemplateHead || - anyChars.currentToken().type == TokenKind::NoSubsTemplate); - const CharT* cur = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.begin + 1); - const CharT* end; - if (anyChars.currentToken().type == TokenKind::TemplateHead) { - // Of the form |`...${| or |}...${| - end = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 2); - } else { - // NO_SUBS_TEMPLATE is of the form |`...`| or |}...`| - end = this->sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 1); - } - - if (!fillCharBufferWithTemplateStringContents(cur, end)) - return nullptr; - - return drainCharBufferIntoAtom(anyChars.cx); - } - private: // This is private because it should only be called by the tokenizer while // tokenizing not by, for example, BytecodeEmitter. bool reportStrictModeError(unsigned errorNumber, ...); void reportInvalidEscapeError(uint32_t offset, InvalidEscapeType type) { switch (type) { case InvalidEscapeType::None:
--- a/js/src/gc/ArenaList.h +++ b/js/src/gc/ArenaList.h @@ -236,17 +236,17 @@ class FreeLists inline void clear(); MOZ_ALWAYS_INLINE TenuredCell* allocate(AllocKind kind); inline TenuredCell* setArenaAndAllocate(Arena* arena, AllocKind kind); inline void unmarkPreMarkedFreeCells(AllocKind kind); - const void* addressOfFreeList(AllocKind thingKind) const { + FreeSpan** addressOfFreeList(AllocKind thingKind) { return &freeLists_[thingKind]; } }; class ArenaLists { JS::Zone* zone_; @@ -293,17 +293,17 @@ class ArenaLists public: explicit ArenaLists(JS::Zone* zone); ~ArenaLists(); FreeLists& freeLists() { return freeLists_.ref(); } const FreeLists& freeLists() const { return freeLists_.ref(); } - const void* addressOfFreeList(AllocKind thingKind) const { + FreeSpan** addressOfFreeList(AllocKind thingKind) { return freeLists_.refNoCheck().addressOfFreeList(thingKind); } inline Arena* getFirstArena(AllocKind thingKind) const; inline Arena* getFirstArenaToSweep(AllocKind thingKind) const; inline Arena* getFirstSweptArena(AllocKind thingKind) const; inline Arena* getArenaAfterCursor(AllocKind thingKind) const;
--- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -288,17 +288,19 @@ class GCRuntime void purgeRuntimeForMinorGC(); void shrinkBuffers(); void onOutOfMallocMemory(); void onOutOfMallocMemory(const AutoLockGC& lock); #ifdef JS_GC_ZEAL - const void* addressOfZealModeBits() { return &zealModeBits; } + const uint32_t* addressOfZealModeBits() { + return &zealModeBits.refNoCheck(); + } void getZealBits(uint32_t* zealBits, uint32_t* frequency, uint32_t* nextScheduled); void setZeal(uint8_t zeal, uint32_t frequency); bool parseAndSetZeal(const char* str); void setNextScheduled(uint32_t count); void verifyPreBarriers(); void maybeVerifyPreBarriers(bool always); bool selectForMarking(JSObject* object); void clearSelectedForMarking(); @@ -1000,17 +1002,17 @@ class GCRuntime public: Nursery& nursery() { return nursery_.ref(); } gc::StoreBuffer& storeBuffer() { return storeBuffer_.ref(); } // Free LIFO blocks are transferred to this allocator before being freed // after minor GC. MainThreadData<LifoAlloc> blocksToFreeAfterMinorGC; - const void* addressOfNurseryPosition() { + void* addressOfNurseryPosition() { return nursery_.refNoCheck().addressOfPosition(); } const void* addressOfNurseryCurrentEnd() { return nursery_.refNoCheck().addressOfCurrentEnd(); } const void* addressOfStringNurseryCurrentEnd() { return nursery_.refNoCheck().addressOfCurrentStringEnd(); }
--- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -319,19 +319,21 @@ class Nursery void renderProfileJSON(JSONPrinter& json) const; /* Print header line for profile times. */ static void printProfileHeader(); /* Print total profile times on shutdown. */ void printTotalProfileTimes(); - void* addressOfCurrentEnd() const { return (void*)¤tEnd_; } - void* addressOfPosition() const { return (void*)&position_; } - void* addressOfCurrentStringEnd() const { return (void*)¤tStringEnd_; } + void* addressOfPosition() const { return (void**)&position_; } + const void* addressOfCurrentEnd() const { return (void**)¤tEnd_; } + const void* addressOfCurrentStringEnd() const { + return (void*)¤tStringEnd_; + } void requestMinorGC(JS::gcreason::Reason reason) const; bool minorGCRequested() const { return minorGCTriggerReason_ != JS::gcreason::NO_REASON; } JS::gcreason::Reason minorGCTriggerReason() const { return minorGCTriggerReason_; } void clearMinorGCRequest() { minorGCTriggerReason_ = JS::gcreason::NO_REASON; } bool needIdleTimeCollection() const;
--- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -4044,18 +4044,19 @@ CodeGenerator::maybeEmitGlobalBarrierChe if (!maybeGlobal->isConstant()) return; JSObject* obj = &maybeGlobal->toConstant()->toObject(); if (gen->realm->maybeGlobal() != obj) return; - auto addr = AbsoluteAddress(gen->realm->addressOfGlobalWriteBarriered()); - masm.branch32(Assembler::NotEqual, addr, Imm32(0), ool->rejoin()); + const uint32_t* addr = gen->realm->addressOfGlobalWriteBarriered(); + masm.branch32(Assembler::NotEqual, AbsoluteAddress(addr), Imm32(0), + ool->rejoin()); } template <class LPostBarrierType, MIRType nurseryType> void CodeGenerator::visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool) { addOutOfLineCode(ool, lir->mir()); @@ -13281,17 +13282,18 @@ CodeGenerator::visitRandom(LRandom* ins) #ifdef JS_PUNBOX64 Register64 s0Reg(ToRegister(ins->temp1())); Register64 s1Reg(ToRegister(ins->temp2())); #else Register64 s0Reg(ToRegister(ins->temp1()), ToRegister(ins->temp2())); Register64 s1Reg(ToRegister(ins->temp3()), ToRegister(ins->temp4())); #endif - const void* rng = gen->realm->addressOfRandomNumberGenerator(); + const XorShift128PlusRNG* rng = + gen->realm->addressOfRandomNumberGenerator(); masm.movePtr(ImmPtr(rng), tempReg); static_assert(sizeof(XorShift128PlusRNG) == 2 * sizeof(uint64_t), "Code below assumes XorShift128PlusRNG contains two uint64_t values"); Address state0Addr(tempReg, XorShift128PlusRNG::offsetOfState0()); Address state1Addr(tempReg, XorShift128PlusRNG::offsetOfState1());
--- a/js/src/jit/CompileWrappers.cpp +++ b/js/src/jit/CompileWrappers.cpp @@ -23,17 +23,17 @@ CompileRuntime::runtime() /* static */ CompileRuntime* CompileRuntime::get(JSRuntime* rt) { return reinterpret_cast<CompileRuntime*>(rt); } #ifdef JS_GC_ZEAL -const void* +const uint32_t* CompileRuntime::addressOfGCZealModeBits() { return runtime()->gc.addressOfZealModeBits(); } #endif const JitRuntime* CompileRuntime::jitRuntime() @@ -166,50 +166,56 @@ CompileZone::isAtomsZone() #ifdef DEBUG const void* CompileZone::addressOfIonBailAfter() { return zone()->runtimeFromAnyThread()->jitRuntime()->addressOfIonBailAfter(); } #endif -const void* +const uint32_t* CompileZone::addressOfNeedsIncrementalBarrier() { return zone()->addressOfNeedsIncrementalBarrier(); } -const void* +gc::FreeSpan** CompileZone::addressOfFreeList(gc::AllocKind allocKind) { return zone()->arenas.addressOfFreeList(allocKind); } -const void* +void* CompileZone::addressOfNurseryPosition() { return zone()->runtimeFromAnyThread()->gc.addressOfNurseryPosition(); } -const void* +void* CompileZone::addressOfStringNurseryPosition() { // Objects and strings share a nursery, for now at least. return zone()->runtimeFromAnyThread()->gc.addressOfNurseryPosition(); } const void* CompileZone::addressOfNurseryCurrentEnd() { return zone()->runtimeFromAnyThread()->gc.addressOfNurseryCurrentEnd(); } const void* CompileZone::addressOfStringNurseryCurrentEnd() { + // Although objects and strings share a nursery (and this may change) + // there is still a separate string end address. The only time it + // is different from the regular end address, is when nursery strings are + // disabled (it will be NULL). + // + // This function returns _a pointer to_ that end address. return zone()->runtimeFromAnyThread()->gc.addressOfStringNurseryCurrentEnd(); } bool CompileZone::canNurseryAllocateStrings() { return nurseryExists() && zone()->runtimeFromAnyThread()->gc.nursery().canAllocateStrings() && @@ -249,17 +255,17 @@ CompileRealm::zone() } CompileRuntime* CompileRealm::runtime() { return CompileRuntime::get(realm()->runtimeFromAnyThread()); } -const void* +const mozilla::non_crypto::XorShift128PlusRNG* CompileRealm::addressOfRandomNumberGenerator() { return realm()->addressOfRandomNumberGenerator(); } const JitRealm* CompileRealm::jitRealm() {
--- a/js/src/jit/CompileWrappers.h +++ b/js/src/jit/CompileWrappers.h @@ -23,17 +23,17 @@ class JitRuntime; class CompileRuntime { JSRuntime* runtime(); public: static CompileRuntime* get(JSRuntime* rt); #ifdef JS_GC_ZEAL - const void* addressOfGCZealModeBits(); + const uint32_t* addressOfGCZealModeBits(); #endif const JitRuntime* jitRuntime(); // Compilation does not occur off thread when the Gecko Profiler is enabled. GeckoProfilerRuntime& geckoProfiler(); bool jitSupportsFloatingPoint(); @@ -70,20 +70,20 @@ class CompileZone CompileRuntime* runtime(); bool isAtomsZone(); #ifdef DEBUG const void* addressOfIonBailAfter(); #endif - const void* addressOfNeedsIncrementalBarrier(); - const void* addressOfFreeList(gc::AllocKind allocKind); - const void* addressOfNurseryPosition(); - const void* addressOfStringNurseryPosition(); + const uint32_t* addressOfNeedsIncrementalBarrier(); + gc::FreeSpan** addressOfFreeList(gc::AllocKind allocKind); + void* addressOfNurseryPosition(); + void* addressOfStringNurseryPosition(); const void* addressOfNurseryCurrentEnd(); const void* addressOfStringNurseryCurrentEnd(); bool nurseryExists(); bool canNurseryAllocateStrings(); void setMinorGCShouldCancelIonCompilations(); }; @@ -98,17 +98,18 @@ class CompileRealm CompileZone* zone(); CompileRuntime* runtime(); const void* realmPtr() { return realm(); } - const void* addressOfRandomNumberGenerator(); + const mozilla::non_crypto::XorShift128PlusRNG* + addressOfRandomNumberGenerator(); const JitRealm* jitRealm(); const GlobalObject* maybeGlobal(); const uint32_t* addressOfGlobalWriteBarriered(); bool hasAllocationMetadataBuilder();
--- a/js/src/jit/MacroAssembler-inl.h +++ b/js/src/jit/MacroAssembler-inl.h @@ -632,18 +632,18 @@ MacroAssembler::branchTestProxyHandlerFa branchPtr(cond, familyAddr, ImmPtr(handlerp), label); } void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond, Label* label) { MOZ_ASSERT(cond == Zero || cond == NonZero); CompileZone* zone = GetJitContext()->realm->zone(); - AbsoluteAddress needsBarrierAddr(zone->addressOfNeedsIncrementalBarrier()); - branchTest32(cond, needsBarrierAddr, Imm32(0x1), label); + const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier(); + branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label); } void MacroAssembler::branchTestMagicValue(Condition cond, const ValueOperand& val, JSWhyMagic why, Label* label) { MOZ_ASSERT(cond == Equal || cond == NotEqual); branchTestValue(cond, val, MagicValue(why), label);
--- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -813,19 +813,20 @@ void MacroAssembler::checkAllocatorState(Label* fail) { // Don't execute the inline path if we are tracing allocations. if (js::gc::gcTracer.traceEnabled()) jump(fail); #ifdef JS_GC_ZEAL // Don't execute the inline path if gc zeal or tracing are active. - branch32(Assembler::NotEqual, - AbsoluteAddress(GetJitContext()->runtime->addressOfGCZealModeBits()), Imm32(0), - fail); + const uint32_t *ptrZealModeBits = + GetJitContext()->runtime->addressOfGCZealModeBits(); + branch32(Assembler::NotEqual, AbsoluteAddress(ptrZealModeBits), Imm32(0), + fail); #endif // Don't execute the inline path if the realm has an object metadata callback, // as the metadata to use for the object may vary between executions of the op. if (GetJitContext()->realm->hasAllocationMetadataBuilder()) jump(fail); } @@ -858,20 +859,23 @@ MacroAssembler::nurseryAllocateObject(Re } // No explicit check for nursery.isEnabled() is needed, as the comparison // with the nursery's end will always fail in such cases. CompileZone* zone = GetJitContext()->realm->zone(); int thingSize = int(gc::Arena::thingSize(allocKind)); int totalSize = thingSize + nDynamicSlots * sizeof(HeapSlot); MOZ_ASSERT(totalSize % gc::CellAlignBytes == 0); - loadPtr(AbsoluteAddress(zone->addressOfNurseryPosition()), result); + void *ptrNurseryPosition = zone->addressOfNurseryPosition(); + loadPtr(AbsoluteAddress(ptrNurseryPosition), result); computeEffectiveAddress(Address(result, totalSize), temp); - branchPtr(Assembler::Below, AbsoluteAddress(zone->addressOfNurseryCurrentEnd()), temp, fail); - storePtr(temp, AbsoluteAddress(zone->addressOfNurseryPosition())); + const void *ptrNurseryCurrentEnd = zone->addressOfNurseryCurrentEnd(); + branchPtr(Assembler::Below, AbsoluteAddress(ptrNurseryCurrentEnd), temp, + fail); + storePtr(temp, AbsoluteAddress(ptrNurseryPosition)); if (nDynamicSlots) { computeEffectiveAddress(Address(result, thingSize), temp); storePtr(temp, Address(result, NativeObject::offsetOfSlots())); } } // Inlined version of FreeSpan::allocate. This does not fill in slots_. @@ -881,35 +885,36 @@ MacroAssembler::freeListAllocate(Registe CompileZone* zone = GetJitContext()->realm->zone(); int thingSize = int(gc::Arena::thingSize(allocKind)); Label fallback; Label success; // Load the first and last offsets of |zone|'s free list for |allocKind|. // If there is no room remaining in the span, fall back to get the next one. - loadPtr(AbsoluteAddress(zone->addressOfFreeList(allocKind)), temp); + gc::FreeSpan **ptrFreeList = zone->addressOfFreeList(allocKind); + loadPtr(AbsoluteAddress(ptrFreeList), temp); load16ZeroExtend(Address(temp, js::gc::FreeSpan::offsetOfFirst()), result); load16ZeroExtend(Address(temp, js::gc::FreeSpan::offsetOfLast()), temp); branch32(Assembler::AboveOrEqual, result, temp, &fallback); // Bump the offset for the next allocation. add32(Imm32(thingSize), result); - loadPtr(AbsoluteAddress(zone->addressOfFreeList(allocKind)), temp); + loadPtr(AbsoluteAddress(ptrFreeList), temp); store16(result, Address(temp, js::gc::FreeSpan::offsetOfFirst())); sub32(Imm32(thingSize), result); addPtr(temp, result); // Turn the offset into a pointer. jump(&success); bind(&fallback); // If there are no free spans left, we bail to finish the allocation. The // interpreter will call the GC allocator to set up a new arena to allocate // from, after which we can resume allocating in the jit. branchTest32(Assembler::Zero, result, result, fail); - loadPtr(AbsoluteAddress(zone->addressOfFreeList(allocKind)), temp); + loadPtr(AbsoluteAddress(ptrFreeList), temp); addPtr(temp, result); // Turn the offset into a pointer. Push(result); // Update the free list to point to the next span (which may be empty). load32(Address(result, 0), result); store32(result, Address(temp, js::gc::FreeSpan::offsetOfFirst())); Pop(result); bind(&success);
--- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -55,36 +55,36 @@ // x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h. // // - If there is no annotation, then there is only one generic definition in // MacroAssembler.cpp. // // - If the declaration is "inline", then the method definition(s) would be in // the "-inl.h" variant of the same file(s). // -// The script check_macroassembler_style.py (check-masm target of the Makefile) -// is used to verify that method definitions are matching the annotation added -// to the method declarations. If there is any difference, then you either -// forgot to define the method in one of the macro assembler, or you forgot to -// update the annotation of the macro assembler declaration. +// The script check_macroassembler_style.py (which runs on every build) is +// used to verify that method definitions match the annotation on the method +// declarations. If there is any difference, then you either forgot to define +// the method in one of the macro assembler, or you forgot to update the +// annotation of the macro assembler declaration. // // Some convenient short-cuts are used to avoid repeating the same list of // architectures on each method declaration, such as PER_ARCH and // PER_SHARED_ARCH. // // Functions that are architecture-agnostic and are the same for all // architectures, that it's necessary to define inline *in this header* to // avoid used-before-defined warnings/errors that would occur if the // definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER // marker at end of the declaration: // // inline uint32_t framePushed() const OOL_IN_HEADER; // // Such functions should then be defined immediately after MacroAssembler's -// definition, for example like so: +// definition, for example: // // //{{{ check_macroassembler_style // inline uint32_t // MacroAssembler::framePushed() const // { // return framePushed_; // } // ////}}} check_macroassembler_style @@ -95,20 +95,20 @@ // * How this macro works: // // DEFINED_ON is a macro which check if, for the current architecture, the // method is defined on the macro assembler or not. // // For each architecture, we have a macro named DEFINED_ON_arch. This macro is // empty if this is not the current architecture. Otherwise it must be either -// set to "define" or "crash" (only use for the none target so-far). +// set to "define" or "crash" (only used for the none target so far). // -// The DEFINED_ON macro maps the list of architecture names given as argument to -// a list of macro names. For example, +// The DEFINED_ON macro maps the list of architecture names given as arguments +// to a list of macro names. For example, // // DEFINED_ON(arm, x86_shared) // // is expanded to // // DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared // // which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS @@ -121,17 +121,17 @@ // crash // // or to nothing, if the current architecture is not listed in the list of // arguments of DEFINED_ON. Note, only one of the DEFINED_ON_arch macro // contributes to the non-empty result, which is the macro of the current // architecture if it is listed in the arguments of DEFINED_ON. // // This result is appended to DEFINED_ON_RESULT_ before expanding the macro, -// which result is either no annotation, a MOZ_CRASH(), or a "= delete" +// which results in either no annotation, a MOZ_CRASH(), or a "= delete" // annotation on the method declaration. # define DEFINED_ON_x86 # define DEFINED_ON_x64 # define DEFINED_ON_x86_shared # define DEFINED_ON_arm # define DEFINED_ON_arm64 # define DEFINED_ON_mips32
--- a/js/src/moz.build +++ b/js/src/moz.build @@ -214,16 +214,17 @@ UNIFIED_SOURCES += [ 'frontend/BytecodeEmitter.cpp', 'frontend/EmitterScope.cpp', 'frontend/FoldConstants.cpp', 'frontend/ForOfLoopControl.cpp', 'frontend/IfEmitter.cpp', 'frontend/JumpList.cpp', 'frontend/NameFunctions.cpp', 'frontend/ParseNode.cpp', + 'frontend/SwitchEmitter.cpp', 'frontend/TDZCheckCache.cpp', 'frontend/TokenStream.cpp', 'frontend/TryEmitter.cpp', 'gc/Allocator.cpp', 'gc/AtomMarking.cpp', 'gc/Barrier.cpp', 'gc/GC.cpp', 'gc/GCTrace.cpp',
new file mode 100644 --- /dev/null +++ b/js/src/tests/non262/regress/regress-1476383-calloc-exc.js @@ -0,0 +1,15 @@ +if (!this.oomTest) { + this.reportCompare && reportCompare(true, true); + quit(0); +} + +let lfPreamble = ` +`; +oomTest(new Function(`var TOTAL_MEMORY = 50 * 1024 * 1024; +HEAP = IHEAP = new Int32Array(TOTAL_MEMORY); +function __Z9makeFastaI10RandomizedEvPKcS2_jRT_(\$id, \$desc, \$N, \$output) +{ +} +`)); + +this.reportCompare && reportCompare(true, true);
--- a/js/src/vm/JSContext.h +++ b/js/src/vm/JSContext.h @@ -172,17 +172,17 @@ struct JSContext : public JS::RootingCon void recoverFromOutOfMemory(); /* * This variation of calloc will call the large-allocation-failure callback * on OOM and retry the allocation. */ template <typename T> T* pod_callocCanGC(size_t numElems, arena_id_t arena = js::MallocArena) { - T* p = pod_calloc<T>(numElems, arena); + T* p = maybe_pod_calloc<T>(numElems, arena); if (MOZ_LIKELY(!!p)) return p; size_t bytes; if (MOZ_UNLIKELY(!js::CalculateAllocSize<T>(numElems, &bytes))) { reportAllocationOverflow(); return nullptr; } p = static_cast<T*>(runtime()->onOutOfMemoryCanGC(js::AllocFunction::Calloc, bytes));
--- a/js/src/vm/Realm.h +++ b/js/src/vm/Realm.h @@ -807,17 +807,18 @@ class JS::Realm : public JS::shadow::Rea // scripts. bool ensureDelazifyScriptsForDebugger(JSContext* cx); void clearBreakpointsIn(js::FreeOp* fop, js::Debugger* dbg, JS::HandleObject handler); // Initializes randomNumberGenerator if needed. mozilla::non_crypto::XorShift128PlusRNG& getOrCreateRandomNumberGenerator(); - const void* addressOfRandomNumberGenerator() const { + const mozilla::non_crypto::XorShift128PlusRNG* + addressOfRandomNumberGenerator() const { return randomNumberGenerator_.ptr(); } mozilla::HashCodeScrambler randomHashCodeScrambler(); bool isAccessValid() const { return validAccessPtr_ ? *validAccessPtr_ : true; }
--- a/mfbt/Utf8.cpp +++ b/mfbt/Utf8.cpp @@ -1,79 +1,39 @@ /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/Maybe.h" +#include "mozilla/TextUtils.h" #include "mozilla/Types.h" #include "mozilla/Utf8.h" #include <stddef.h> #include <stdint.h> MFBT_API bool mozilla::IsValidUtf8(const void* aCodeUnits, size_t aCount) { const auto* s = static_cast<const unsigned char*>(aCodeUnits); - const auto* limit = s + aCount; + const auto* const limit = s + aCount; while (s < limit) { - uint32_t n = *s++; + unsigned char c = *s++; // If the first byte is ASCII, it's the only one in the code point. Have a // fast path that avoids all the rest of the work and looping in that case. - if ((n & 0x80) == 0) { + if (IsAscii(c)) { continue; } - // The leading code unit determines the length of the next code point and - // the number of bits of the leading code unit that contribute to the code - // point's value. - uint_fast8_t remaining; - uint32_t min; - if ((n & 0xE0) == 0xC0) { - remaining = 1; - min = 0x80; - n &= 0x1F; - } else if ((n & 0xF0) == 0xE0) { - remaining = 2; - min = 0x800; - n &= 0x0F; - } else if ((n & 0xF8) == 0xF0) { - remaining = 3; - min = 0x10000; - n &= 0x07; - } else { - // UTF-8 used to have a hyper-long encoding form, but it's been removed - // for years now. So in this case, the string is not valid UTF-8. + Maybe<char32_t> maybeCodePoint = + DecodeOneUtf8CodePoint(Utf8Unit(c), &s, limit); + if (maybeCodePoint.isNothing()) { return false; } - - // If the code point would require more code units than remain, the encoding - // is invalid. - if (s + remaining > limit) { - return false; - } - - for (uint_fast8_t i = 0; i < remaining; i++) { - // Every non-leading code unit in properly encoded UTF-8 has its high bit - // set and the next-highest bit unset. - if ((s[i] & 0xC0) != 0x80) { - return false; - } - - // The code point being encoded is the concatenation of all the - // unconstrained bits. - n = (n << 6) | (s[i] & 0x3F); - } - - // Don't consider code points that are overlong, UTF-16 surrogates, or - // exceed the maximum code point to be valid. - if (n < min || (0xD800 <= n && n < 0xE000) || n >= 0x110000) { - return false; - } - - s += remaining; } + MOZ_ASSERT(s == limit); return true; }
--- a/mfbt/Utf8.h +++ b/mfbt/Utf8.h @@ -7,16 +7,20 @@ /* * UTF-8-related functionality, including a type-safe structure representing a * UTF-8 code unit. */ #ifndef mozilla_Utf8_h #define mozilla_Utf8_h +#include "mozilla/Casting.h" // for mozilla::AssertedCast +#include "mozilla/Likely.h" // for MOZ_UNLIKELY +#include "mozilla/Maybe.h" // for mozilla::Maybe +#include "mozilla/TextUtils.h" // for mozilla::IsAscii #include "mozilla/Types.h" // for MFBT_API #include <limits.h> // for CHAR_BIT #include <stddef.h> // for size_t #include <stdint.h> // for uint8_t namespace mozilla { @@ -200,11 +204,222 @@ public: * * A valid UTF-8 string contains no overlong-encoded code points (as one would * expect) and contains no code unit sequence encoding a UTF-16 surrogate. The * string *may* contain U+0000 NULL code points. */ extern MFBT_API bool IsValidUtf8(const void* aCodeUnits, size_t aCount); +/** + * Given |aLeadUnit| that is a non-ASCII code unit, a pointer to an |Iter aIter| + * that (initially) itself points one unit past |aLeadUnit|, and + * |const EndIter aEnd| that denotes the end of the UTF-8 data when compared + * against |*aIter| using |aEnd - *aIter|: + * + * If |aLeadUnit| and subsequent code units computed using |*aIter| (up to + * |aEnd|) encode a valid code point -- not exceeding Unicode's range, not a + * surrogate, in shortest form -- then return Some(that code point) and advance + * |*aIter| past those code units. + * + * Otherwise decrement |*aIter| (so that it points at |aLeadUnit|) and return + * Nothing(). + * + * |Iter| and |EndIter| are generalized concepts most easily understood as if + * they were |const char*|, |const unsigned char*|, or |const Utf8Unit*|: + * iterators that when dereferenced can be used to construct a |Utf8Unit| and + * that can be compared and modified in certain limited ways. (Carefully note + * that this function mutates |*aIter|.) |Iter| and |EndIter| are template + * parameters to support more-complicated adaptor iterators. + * + * The template parameters after |Iter| allow users to implement custom handling + * for various forms of invalid UTF-8. A version of this function that defaults + * all such handling to no-ops is defined below this function. To learn how to + * define your own custom handling, consult the implementation of that function, + * which documents exactly how custom handler functors are invoked. + * + * This function is MOZ_ALWAYS_INLINE: if you don't need that, use the version + * of this function without the "Inline" suffix on the name. + */ +template<typename Iter, + typename EndIter, + class OnBadLeadUnit, + class OnNotEnoughUnits, + class OnBadTrailingUnit, + class OnBadCodePoint, + class OnNotShortestForm> +MOZ_ALWAYS_INLINE Maybe<char32_t> +DecodeOneUtf8CodePointInline(const Utf8Unit aLeadUnit, + Iter* aIter, const EndIter aEnd, + OnBadLeadUnit aOnBadLeadUnit, + OnNotEnoughUnits aOnNotEnoughUnits, + OnBadTrailingUnit aOnBadTrailingUnit, + OnBadCodePoint aOnBadCodePoint, + OnNotShortestForm aOnNotShortestForm) +{ + MOZ_ASSERT(Utf8Unit((*aIter)[-1]) == aLeadUnit); + + char32_t n = aLeadUnit.toUint8(); + MOZ_ASSERT(!IsAscii(n)); + + // |aLeadUnit| determines the number of trailing code units in the code point + // and the bits of |aLeadUnit| that contribute to the code point's value. + uint8_t remaining; + uint32_t min; + if ((n & 0b1110'0000) == 0b1100'0000) { + remaining = 1; + min = 0x80; + n &= 0b0001'1111; + } else if ((n & 0b1111'0000) == 0b1110'0000) { + remaining = 2; + min = 0x800; + n &= 0b0000'1111; + } else if ((n & 0b1111'1000) == 0b1111'0000) { + remaining = 3; + min = 0x10000; + n &= 0b0000'0111; + } else { + *aIter -= 1; + aOnBadLeadUnit(); + return Nothing(); + } + + // If the code point would require more code units than remain, the encoding + // is invalid. + auto actual = aEnd - *aIter; + if (MOZ_UNLIKELY(actual < remaining)) { + *aIter -= 1; + aOnNotEnoughUnits(AssertedCast<uint8_t>(actual + 1), remaining + 1); + return Nothing(); + } + + for (uint8_t i = 0; i < remaining; i++) { + uint8_t unit = Utf8Unit(*(*aIter)++).toUint8(); + + // Every non-leading code unit in properly encoded UTF-8 has its high + // bit set and the next-highest bit unset. + if (MOZ_UNLIKELY((unit & 0b1100'0000) != 0b1000'0000)) { + uint8_t unitsObserved = i + 1 + 1; + *aIter -= unitsObserved; + aOnBadTrailingUnit(unitsObserved); + return Nothing(); + } + + // The code point being encoded is the concatenation of all the + // unconstrained bits. + n = (n << 6) | (unit & 0b0011'1111); + } + + // UTF-16 surrogates and values outside the Unicode range are invalid. + if (MOZ_UNLIKELY(n > 0x10FFFF || (0xD800 <= n && n <= 0xDFFF))) { + uint8_t unitsObserved = remaining + 1; + *aIter -= unitsObserved; + aOnBadCodePoint(n, unitsObserved); + return Nothing(); + } + + // Overlong code points are also invalid. + if (MOZ_UNLIKELY(n < min)) { + uint8_t unitsObserved = remaining + 1; + *aIter -= unitsObserved; + aOnNotShortestForm(n, unitsObserved); + return Nothing(); + } + + return Some(n); +} + +/** + * Identical to the above function, but not forced to be instantiated inline -- + * the compiler is permitted to common up separate invocations if it chooses. + */ +template<typename Iter, + typename EndIter, + class OnBadLeadUnit, + class OnNotEnoughUnits, + class OnBadTrailingUnit, + class OnBadCodePoint, + class OnNotShortestForm> +inline Maybe<char32_t> +DecodeOneUtf8CodePoint(const Utf8Unit aLeadUnit, + Iter* aIter, const EndIter aEnd, + OnBadLeadUnit aOnBadLeadUnit, + OnNotEnoughUnits aOnNotEnoughUnits, + OnBadTrailingUnit aOnBadTrailingUnit, + OnBadCodePoint aOnBadCodePoint, + OnNotShortestForm aOnNotShortestForm) +{ + return DecodeOneUtf8CodePointInline(aLeadUnit, aIter, aEnd, + aOnBadLeadUnit, aOnNotEnoughUnits, + aOnBadTrailingUnit, aOnBadCodePoint, + aOnNotShortestForm); +} + +/** + * Like the always-inlined function above, but with no-op behavior from all + * trailing if-invalid notifier functors. + * + * This function is MOZ_ALWAYS_INLINE: if you don't need that, use the version + * of this function without the "Inline" suffix on the name. + */ +template<typename Iter, typename EndIter> +MOZ_ALWAYS_INLINE Maybe<char32_t> +DecodeOneUtf8CodePointInline(const Utf8Unit aLeadUnit, + Iter* aIter, const EndIter aEnd) +{ + // aOnBadLeadUnit is called when |aLeadUnit| itself is an invalid lead unit in + // a multi-unit code point. It is passed no arguments: the caller already has + // |aLeadUnit| on hand, so no need to provide it again. + auto onBadLeadUnit = []() {}; + + // aOnNotEnoughUnits is called when |aLeadUnit| properly indicates a code + // point length, but there aren't enough units from |*aIter| to |aEnd| to + // satisfy that length. It is passed the number of code units actually + // available (according to |aEnd - *aIter|) and the number of code units that + // |aLeadUnit| indicates are needed. Both numbers include the contribution + // of |aLeadUnit| itself: so |aUnitsAvailable <= 3|, |aUnitsNeeded <= 4|, and + // |aUnitsAvailable < aUnitsNeeded|. As above, it also is not passed the lead + // code unit. + auto onNotEnoughUnits = [](uint8_t aUnitsAvailable, uint8_t aUnitsNeeded) {}; + + // aOnBadTrailingUnit is called when one of the trailing code units implied by + // |aLeadUnit| doesn't match the 0b10xx'xxxx bit pattern that all UTF-8 + // trailing code units must satisfy. It is passed the total count of units + // observed (including |aLeadUnit|). The bad trailing code unit will + // conceptually be at |(*aIter)[aUnitsObserved - 1]| if this functor is + // called, and so |aUnitsObserved <= 4|. + auto onBadTrailingUnit = [](uint8_t aUnitsObserved) {}; + + // aOnBadCodePoint is called when a structurally-correct code point encoding + // is found, but the *value* that is encoded is not a valid code point: either + // because it exceeded the U+10FFFF Unicode maximum code point, or because it + // was a UTF-16 surrogate. It is passed the non-code point value and the + // number of code units used to encode it. + auto onBadCodePoint = [](char32_t aBadCodePoint, uint8_t aUnitsObserved) {}; + + // aOnNotShortestForm is called when structurally-correct encoding is found, + // but the encoded value should have been encoded in fewer code units (e.g. + // mis-encoding U+0000 as 0b1100'0000 0b1000'0000 in two code units instead of + // as 0b0000'0000). It is passed the mis-encoded code point (which will be + // valid and not a surrogate) and the count of code units that mis-encoded it. + auto onNotShortestForm = [](char32_t aBadCodePoint, uint8_t aUnitsObserved) {}; + + return DecodeOneUtf8CodePointInline(aLeadUnit, aIter, aEnd, + onBadLeadUnit, onNotEnoughUnits, + onBadTrailingUnit, onBadCodePoint, + onNotShortestForm); +} + +/** + * Identical to the above function, but not forced to be instantiated inline -- + * the compiler/linker are allowed to common up separate invocations. + */ +template<typename Iter, typename EndIter> +inline Maybe<char32_t> +DecodeOneUtf8CodePoint(const Utf8Unit aLeadUnit, + Iter* aIter, const EndIter aEnd) +{ + return DecodeOneUtf8CodePointInline(aLeadUnit, aIter, aEnd); +} + } // namespace mozilla #endif /* mozilla_Utf8_h */
--- a/mfbt/tests/TestUtf8.cpp +++ b/mfbt/tests/TestUtf8.cpp @@ -3,18 +3,25 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/Utf8.h" #include "mozilla/ArrayUtils.h" #include "mozilla/Assertions.h" +#include "mozilla/EnumSet.h" +#include "mozilla/IntegerRange.h" +#include "mozilla/TextUtils.h" using mozilla::ArrayLength; +using mozilla::DecodeOneUtf8CodePoint; +using mozilla::EnumSet; +using mozilla::IntegerRange; +using mozilla::IsAscii; using mozilla::IsValidUtf8; using mozilla::Utf8Unit; static void TestUtf8Unit() { Utf8Unit c('A'); MOZ_RELEASE_ASSERT(c.toChar() == 'A'); @@ -30,16 +37,252 @@ TestUtf8Unit() Utf8Unit second('#'); MOZ_RELEASE_ASSERT(first != second); first = second; MOZ_RELEASE_ASSERT(first == second); } +template<typename Char> +struct ToUtf8Units +{ +public: + explicit ToUtf8Units(const Char* aStart, const Char* aEnd) + : lead(Utf8Unit(aStart[0])) + , iter(aStart + 1) + , end(aEnd) + { + MOZ_RELEASE_ASSERT(!IsAscii(aStart[0])); + } + + const Utf8Unit lead; + const Char* iter; + const Char* const end; +}; + +class AssertIfCalled +{ +public: + template<typename... Args> + void operator()(Args&&... aArgs) { + MOZ_RELEASE_ASSERT(false, "AssertIfCalled instance was called"); + } +}; + +// NOTE: For simplicity in treating |aCharN| identically regardless whether it's +// a string literal or a more-generalized array, we require |aCharN| be +// null-terminated. + +template<typename Char, size_t N> +static void +ExpectValidCodePoint(const Char (&aCharN)[N], + char32_t aExpectedCodePoint) +{ + MOZ_RELEASE_ASSERT(aCharN[N - 1] == 0, + "array must be null-terminated for |aCharN + N - 1| to " + "compute the value of |aIter| as altered by " + "DecodeOneUtf8CodePoint"); + + ToUtf8Units<Char> simpleUnit(aCharN, aCharN + N - 1); + auto simple = + DecodeOneUtf8CodePoint(simpleUnit.lead, &simpleUnit.iter, simpleUnit.end); + MOZ_RELEASE_ASSERT(simple.isSome()); + MOZ_RELEASE_ASSERT(*simple == aExpectedCodePoint); + MOZ_RELEASE_ASSERT(simpleUnit.iter == simpleUnit.end); + + ToUtf8Units<Char> complexUnit(aCharN, aCharN + N - 1); + auto complex = + DecodeOneUtf8CodePoint(complexUnit.lead, &complexUnit.iter, complexUnit.end, + AssertIfCalled(), + AssertIfCalled(), + AssertIfCalled(), + AssertIfCalled(), + AssertIfCalled()); + MOZ_RELEASE_ASSERT(complex.isSome()); + MOZ_RELEASE_ASSERT(*complex == aExpectedCodePoint); + MOZ_RELEASE_ASSERT(complexUnit.iter == complexUnit.end); +} + +enum class InvalidUtf8Reason +{ + BadLeadUnit, + NotEnoughUnits, + BadTrailingUnit, + BadCodePoint, + NotShortestForm, +}; + +template<typename Char, size_t N> +static void +ExpectInvalidCodePointHelper(const Char (&aCharN)[N], + InvalidUtf8Reason aExpectedReason, + uint8_t aExpectedUnitsAvailable, + uint8_t aExpectedUnitsNeeded, + char32_t aExpectedBadCodePoint, + uint8_t aExpectedUnitsObserved) +{ + MOZ_RELEASE_ASSERT(aCharN[N - 1] == 0, + "array must be null-terminated for |aCharN + N - 1| to " + "compute the value of |aIter| as altered by " + "DecodeOneUtf8CodePoint"); + + ToUtf8Units<Char> simpleUnit(aCharN, aCharN + N - 1); + auto simple = + DecodeOneUtf8CodePoint(simpleUnit.lead, &simpleUnit.iter, simpleUnit.end); + MOZ_RELEASE_ASSERT(simple.isNothing()); + MOZ_RELEASE_ASSERT(static_cast<const void*>(simpleUnit.iter) == aCharN); + + EnumSet<InvalidUtf8Reason> reasons; + uint8_t unitsAvailable; + uint8_t unitsNeeded; + char32_t badCodePoint; + uint8_t unitsObserved; + + struct OnNotShortestForm + { + EnumSet<InvalidUtf8Reason>& reasons; + char32_t& badCodePoint; + uint8_t& unitsObserved; + + void operator()(char32_t aBadCodePoint, uint8_t aUnitsObserved) { + reasons += InvalidUtf8Reason::NotShortestForm; + badCodePoint = aBadCodePoint; + unitsObserved = aUnitsObserved; + } + }; + + ToUtf8Units<Char> complexUnit(aCharN, aCharN + N - 1); + auto complex = + DecodeOneUtf8CodePoint(complexUnit.lead, &complexUnit.iter, complexUnit.end, + [&reasons]() { + reasons += InvalidUtf8Reason::BadLeadUnit; + }, + [&reasons, &unitsAvailable, &unitsNeeded](uint8_t aUnitsAvailable, + uint8_t aUnitsNeeded) + { + reasons += InvalidUtf8Reason::NotEnoughUnits; + unitsAvailable = aUnitsAvailable; + unitsNeeded = aUnitsNeeded; + }, + [&reasons, &unitsObserved](uint8_t aUnitsObserved) + { + reasons += InvalidUtf8Reason::BadTrailingUnit; + unitsObserved = aUnitsObserved; + }, + [&reasons, &badCodePoint, &unitsObserved](char32_t aBadCodePoint, + uint8_t aUnitsObserved) + { + reasons += InvalidUtf8Reason::BadCodePoint; + badCodePoint = aBadCodePoint; + unitsObserved = aUnitsObserved; + }, + [&reasons, &badCodePoint, &unitsObserved](char32_t aBadCodePoint, + uint8_t aUnitsObserved) + { + reasons += InvalidUtf8Reason::NotShortestForm; + badCodePoint = aBadCodePoint; + unitsObserved = aUnitsObserved; + }); + MOZ_RELEASE_ASSERT(complex.isNothing()); + MOZ_RELEASE_ASSERT(static_cast<const void*>(complexUnit.iter) == aCharN); + + bool alreadyIterated = false; + for (InvalidUtf8Reason reason : reasons) { + MOZ_RELEASE_ASSERT(!alreadyIterated); + alreadyIterated = true; + + switch (reason) { + case InvalidUtf8Reason::BadLeadUnit: + break; + + case InvalidUtf8Reason::NotEnoughUnits: + MOZ_RELEASE_ASSERT(unitsAvailable == aExpectedUnitsAvailable); + MOZ_RELEASE_ASSERT(unitsNeeded == aExpectedUnitsNeeded); + break; + + case InvalidUtf8Reason::BadTrailingUnit: + MOZ_RELEASE_ASSERT(unitsObserved == aExpectedUnitsObserved); + break; + + case InvalidUtf8Reason::BadCodePoint: + MOZ_RELEASE_ASSERT(badCodePoint == aExpectedBadCodePoint); + MOZ_RELEASE_ASSERT(unitsObserved == aExpectedUnitsObserved); + break; + + case InvalidUtf8Reason::NotShortestForm: + MOZ_RELEASE_ASSERT(badCodePoint == aExpectedBadCodePoint); + MOZ_RELEASE_ASSERT(unitsObserved == aExpectedUnitsObserved); + break; + } + } +} + +// NOTE: For simplicity in treating |aCharN| identically regardless whether it's +// a string literal or a more-generalized array, we require |aCharN| be +// null-terminated in all these functions. + +template<typename Char, size_t N> +static void +ExpectBadLeadUnit(const Char (&aCharN)[N]) +{ + ExpectInvalidCodePointHelper(aCharN, + InvalidUtf8Reason::BadLeadUnit, + 0xFF, 0xFF, 0xFFFFFFFF, 0xFF); +} + +template<typename Char, size_t N> +static void +ExpectNotEnoughUnits(const Char (&aCharN)[N], + uint8_t aExpectedUnitsAvailable, + uint8_t aExpectedUnitsNeeded) +{ + ExpectInvalidCodePointHelper(aCharN, + InvalidUtf8Reason::NotEnoughUnits, + aExpectedUnitsAvailable, aExpectedUnitsNeeded, + 0xFFFFFFFF, 0xFF); +} + +template<typename Char, size_t N> +static void +ExpectBadTrailingUnit(const Char (&aCharN)[N], + uint8_t aExpectedUnitsObserved) +{ + ExpectInvalidCodePointHelper(aCharN, + InvalidUtf8Reason::BadTrailingUnit, + 0xFF, 0xFF, 0xFFFFFFFF, + aExpectedUnitsObserved); +} + +template<typename Char, size_t N> +static void +ExpectNotShortestForm(const Char (&aCharN)[N], + char32_t aExpectedBadCodePoint, + uint8_t aExpectedUnitsObserved) +{ + ExpectInvalidCodePointHelper(aCharN, + InvalidUtf8Reason::NotShortestForm, + 0xFF, 0xFF, + aExpectedBadCodePoint, + aExpectedUnitsObserved); +} + +template<typename Char, size_t N> +static void +ExpectBadCodePoint(const Char (&aCharN)[N], + char32_t aExpectedBadCodePoint, + uint8_t aExpectedUnitsObserved) +{ + ExpectInvalidCodePointHelper(aCharN, + InvalidUtf8Reason::BadCodePoint, + 0xFF, 0xFF, + aExpectedBadCodePoint, + aExpectedUnitsObserved); +} + static void TestIsValidUtf8() { // Note we include the U+0000 NULL in this one -- and that's fine. static const char asciiBytes[] = u8"How about a nice game of chess?"; MOZ_RELEASE_ASSERT(IsValidUtf8(asciiBytes, ArrayLength(asciiBytes))); static const char endNonAsciiBytes[] = u8"Life is like a 🌯"; @@ -57,59 +300,481 @@ TestIsValidUtf8() MOZ_RELEASE_ASSERT(IsValidUtf8(oneBytes, oneBytesLen)); // 2 static const char twoBytes[] = u8"؆"; // U+0606 ARABIC-INDIC CUBE ROOT constexpr size_t twoBytesLen = ArrayLength(twoBytes); static_assert(twoBytesLen == 3, "U+0606 in two bytes plus nul"); MOZ_RELEASE_ASSERT(IsValidUtf8(twoBytes, twoBytesLen)); + ExpectValidCodePoint(twoBytes, 0x0606); + // 3 static const char threeBytes[] = u8"᨞"; // U+1A1E BUGINESE PALLAWA constexpr size_t threeBytesLen = ArrayLength(threeBytes); static_assert(threeBytesLen == 4, "U+1A1E in three bytes plus nul"); MOZ_RELEASE_ASSERT(IsValidUtf8(threeBytes, threeBytesLen)); + ExpectValidCodePoint(threeBytes, 0x1A1E); + // 4 static const char fourBytes[] = u8"🁡"; // U+1F061 DOMINO TILE HORIZONTAL-06-06 constexpr size_t fourBytesLen = ArrayLength(fourBytes); static_assert(fourBytesLen == 5, "U+1F061 in four bytes plus nul"); MOZ_RELEASE_ASSERT(IsValidUtf8(fourBytes, fourBytesLen)); + ExpectValidCodePoint(fourBytes, 0x1F061); + // Max code point static const char maxCodePoint[] = u8""; // U+10FFFF constexpr size_t maxCodePointLen = ArrayLength(maxCodePoint); static_assert(maxCodePointLen == 5, "U+10FFFF in four bytes plus nul"); MOZ_RELEASE_ASSERT(IsValidUtf8(maxCodePoint, maxCodePointLen)); + ExpectValidCodePoint(maxCodePoint, 0x10FFFF); + // One past max code point - static unsigned const char onePastMaxCodePoint[] = { 0xF4, 0x90, 0x80, 0x80 }; + static const unsigned char onePastMaxCodePoint[] = { 0xF4, 0x90, 0x80, 0x80, 0x0 }; constexpr size_t onePastMaxCodePointLen = ArrayLength(onePastMaxCodePoint); MOZ_RELEASE_ASSERT(!IsValidUtf8(onePastMaxCodePoint, onePastMaxCodePointLen)); + ExpectBadCodePoint(onePastMaxCodePoint, 0x110000, 4); + // Surrogate-related testing - static const unsigned char justBeforeSurrogates[] = { 0xED, 0x9F, 0xBF }; - MOZ_RELEASE_ASSERT(IsValidUtf8(justBeforeSurrogates, ArrayLength(justBeforeSurrogates))); + // (Note that the various code unit sequences here are null-terminated to + // simplify life for ExpectValidCodePoint, which presumes null termination.) + + static const unsigned char justBeforeSurrogates[] = { 0xED, 0x9F, 0xBF, 0x0 }; + constexpr size_t justBeforeSurrogatesLen = ArrayLength(justBeforeSurrogates) - 1; + MOZ_RELEASE_ASSERT(IsValidUtf8(justBeforeSurrogates, justBeforeSurrogatesLen)); + + ExpectValidCodePoint(justBeforeSurrogates, 0xD7FF); + + static const unsigned char leastSurrogate[] = { 0xED, 0xA0, 0x80, 0x0 }; + constexpr size_t leastSurrogateLen = ArrayLength(leastSurrogate) - 1; + MOZ_RELEASE_ASSERT(!IsValidUtf8(leastSurrogate, leastSurrogateLen)); + + ExpectBadCodePoint(leastSurrogate, 0xD800, 3); + + static const unsigned char arbitraryHighSurrogate[] = { 0xED, 0xA2, 0x87, 0x0 }; + constexpr size_t arbitraryHighSurrogateLen = ArrayLength(arbitraryHighSurrogate) - 1; + MOZ_RELEASE_ASSERT(!IsValidUtf8(arbitraryHighSurrogate, arbitraryHighSurrogateLen)); + + ExpectBadCodePoint(arbitraryHighSurrogate, 0xD887, 3); + + static const unsigned char arbitraryLowSurrogate[] = { 0xED, 0xB7, 0xAF, 0x0 }; + constexpr size_t arbitraryLowSurrogateLen = ArrayLength(arbitraryLowSurrogate) - 1; + MOZ_RELEASE_ASSERT(!IsValidUtf8(arbitraryLowSurrogate, arbitraryLowSurrogateLen)); + + ExpectBadCodePoint(arbitraryLowSurrogate, 0xDDEF, 3); + + static const unsigned char greatestSurrogate[] = { 0xED, 0xBF, 0xBF, 0x0 }; + constexpr size_t greatestSurrogateLen = ArrayLength(greatestSurrogate) - 1; + MOZ_RELEASE_ASSERT(!IsValidUtf8(greatestSurrogate, greatestSurrogateLen)); + + ExpectBadCodePoint(greatestSurrogate, 0xDFFF, 3); + + static const unsigned char justAfterSurrogates[] = { 0xEE, 0x80, 0x80, 0x0 }; + constexpr size_t justAfterSurrogatesLen = ArrayLength(justAfterSurrogates) - 1; + MOZ_RELEASE_ASSERT(IsValidUtf8(justAfterSurrogates, justAfterSurrogatesLen)); + + ExpectValidCodePoint(justAfterSurrogates, 0xE000); +} + +static void +TestDecodeOneValidUtf8CodePoint() +{ + // NOTE: DecodeOneUtf8CodePoint decodes only *non*-ASCII code points that + // consist of multiple code units, so there are no ASCII tests below. + + // Length two. + + ExpectValidCodePoint(u8"", 0x80); // <control> + ExpectValidCodePoint(u8"©", 0xA9); // COPYRIGHT SIGN + ExpectValidCodePoint(u8"¶", 0xB6); // PILCROW SIGN + ExpectValidCodePoint(u8"¾", 0xBE); // VULGAR FRACTION THREE QUARTERS + ExpectValidCodePoint(u8"÷", 0xF7); // DIVISION SIGN + ExpectValidCodePoint(u8"ÿ", 0xFF); // LATIN SMALL LETTER Y WITH DIAERESIS + ExpectValidCodePoint(u8"Ā", 0x100); // LATIN CAPITAL LETTER A WITH MACRON + ExpectValidCodePoint(u8"IJ", 0x132); // LATIN CAPITAL LETTER LIGATURE IJ + ExpectValidCodePoint(u8"ͼ", 0x37C); // GREEK SMALL DOTTED LUNATE SIGMA SYMBOL + ExpectValidCodePoint(u8"Ӝ", 0x4DC); // CYRILLIC CAPITAL LETTER ZHE WITTH DIAERESIS + ExpectValidCodePoint(u8"۩", 0x6E9); // ARABIC PLACE OF SAJDAH + ExpectValidCodePoint(u8"߿", 0x7FF); // <not assigned> + + // Length three. + + ExpectValidCodePoint(u8"ࠀ", 0x800); // SAMARITAN LETTER ALAF + ExpectValidCodePoint(u8"ࡁ", 0x841); // MANDAIC LETTER AB + ExpectValidCodePoint(u8"ࣿ", 0x8FF); // ARABIC MARK SIDEWAYS NOON GHUNNA + ExpectValidCodePoint(u8"ஆ", 0xB86); // TAMIL LETTER AA + ExpectValidCodePoint(u8"༃", 0xF03); // TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA + ExpectValidCodePoint(u8"࿉", 0xFC9); // TIBETAN SYMBOL NOR BU (but on my system it really looks like SOFT-SERVE ICE CREAM FROM ABOVE THE PLANE if you ask me) + ExpectValidCodePoint(u8"ဪ", 0x102A); // MYANMAR LETTER AU + ExpectValidCodePoint(u8"ᚏ", 0x168F); // OGHAM LETTER RUIS + ExpectValidCodePoint("\xE2\x80\xA8", 0x2028); // (the hated) LINE SEPARATOR + ExpectValidCodePoint("\xE2\x80\xA9", 0x2029); // (the hated) PARAGRAPH SEPARATOR + ExpectValidCodePoint(u8"☬", 0x262C); // ADI SHAKTI + ExpectValidCodePoint(u8"㊮", 0x32AE); // CIRCLED IDEOGRAPH RESOURCE + ExpectValidCodePoint(u8"㏖", 0x33D6); // SQUARE MOL + ExpectValidCodePoint(u8"ꔄ", 0xA504); // VAI SYLLABLE WEEN + ExpectValidCodePoint(u8"ퟕ", 0xD7D5); // HANGUL JONGSEONG RIEUL-SSANGKIYEOK + ExpectValidCodePoint(u8"", 0xD7FF); // <not assigned> + ExpectValidCodePoint(u8"", 0xE000); // <Private Use> + ExpectValidCodePoint(u8"鱗", 0xF9F2); // CJK COMPATIBILITY IDEOGRAPH-F9F + ExpectValidCodePoint(u8"﷽", 0xFDFD); // ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHHHEEEEM + ExpectValidCodePoint(u8"", 0xFFFF); // <not assigned> + + // Length four. + ExpectValidCodePoint(u8"𐀀", 0x10000); // LINEAR B SYLLABLE B008 A + ExpectValidCodePoint(u8"𔑀", 0x14440); // ANATOLIAN HIEROGLYPH A058 + ExpectValidCodePoint(u8"𝛗", 0x1D6D7); // MATHEMATICAL BOLD SMALL PHI + ExpectValidCodePoint(u8"💩", 0x1F4A9); // PILE OF POO + ExpectValidCodePoint(u8"🔫", 0x1F52B); // PISTOL + ExpectValidCodePoint(u8"🥌", 0x1F94C); // CURLING STONE + ExpectValidCodePoint(u8"🥏", 0x1F94F); // FLYING DISC + ExpectValidCodePoint(u8"𠍆", 0x20346); // CJK UNIFIED IDEOGRAPH-20346 + ExpectValidCodePoint(u8"𡠺", 0x2183A); // CJK UNIFIED IDEOGRAPH-2183A + ExpectValidCodePoint(u8"", 0x417F6); // <not assigned> + ExpectValidCodePoint(u8"", 0x7E836); // <not assigned> + ExpectValidCodePoint(u8"", 0xFEF67); // <Plane 15 Private Use> + ExpectValidCodePoint(u8"", 0x10FFFF); // +} + +static void +TestDecodeBadLeadUnit() +{ + // These tests are actually exhaustive. - static const unsigned char leastSurrogate[] = { 0xED, 0xA0, 0x80 }; - MOZ_RELEASE_ASSERT(!IsValidUtf8(leastSurrogate, ArrayLength(leastSurrogate))); + unsigned char badLead[] = { '\0', '\0' }; + + for (uint8_t lead : IntegerRange(0b1000'0000, 0b1100'0000)) { + badLead[0] = lead; + ExpectBadLeadUnit(badLead); + } + + { + uint8_t lead = 0b1111'1000; + do { + badLead[0] = lead; + ExpectBadLeadUnit(badLead); + if (lead == 0b1111'1111) { + break; + } + + lead++; + } while (true); + } +} + +static void +TestTooFewOrBadTrailingUnits() +{ + // Lead unit indicates a two-byte code point. + + char truncatedTwo[] = { '\0', '\0' }; + char badTrailTwo[] = { '\0', '\0', '\0' }; + + for (uint8_t lead : IntegerRange(0b1100'0000, 0b1110'0000)) { + truncatedTwo[0] = lead; + ExpectNotEnoughUnits(truncatedTwo, 1, 2); + + badTrailTwo[0] = lead; + for (uint8_t trail : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailTwo[1] = trail; + ExpectBadTrailingUnit(badTrailTwo, 2); + } + + for (uint8_t trail : IntegerRange(0b1100'0000, 0b1111'1111)) { + badTrailTwo[1] = trail; + ExpectBadTrailingUnit(badTrailTwo, 2); + } + } + + // Lead unit indicates a three-byte code point. + + char truncatedThreeOne[] = { '\0', '\0' }; + char truncatedThreeTwo[] = { '\0', '\0', '\0' }; + unsigned char badTrailThree[] = { '\0', '\0', '\0', '\0' }; - static const unsigned char arbitraryHighSurrogate[] = { 0xED, 0xA2, 0x87 }; - MOZ_RELEASE_ASSERT(!IsValidUtf8(arbitraryHighSurrogate, ArrayLength(arbitraryHighSurrogate))); + for (uint8_t lead : IntegerRange(0b1110'0000, 0b1111'0000)) { + truncatedThreeOne[0] = lead; + ExpectNotEnoughUnits(truncatedThreeOne, 1, 3); + + truncatedThreeTwo[0] = lead; + ExpectNotEnoughUnits(truncatedThreeTwo, 2, 3); + + badTrailThree[0] = lead; + badTrailThree[2] = 0b1011'1111; // make valid to test overreads + for (uint8_t mid : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailThree[1] = mid; + ExpectBadTrailingUnit(badTrailThree, 2); + } + { + uint8_t mid = 0b1100'0000; + do { + badTrailThree[1] = mid; + ExpectBadTrailingUnit(badTrailThree, 2); + if (mid == 0b1111'1111) { + break; + } + + mid++; + } while (true); + } + + badTrailThree[1] = 0b1011'1111; + for (uint8_t last : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailThree[2] = last; + ExpectBadTrailingUnit(badTrailThree, 3); + } + { + uint8_t last = 0b1100'0000; + do { + badTrailThree[2] = last; + ExpectBadTrailingUnit(badTrailThree, 3); + if (last == 0b1111'1111) { + break; + } + + last++; + } while (true); + } + } + + // Lead unit indicates a four-byte code point. + + char truncatedFourOne[] = { '\0', '\0' }; + char truncatedFourTwo[] = { '\0', '\0', '\0' }; + char truncatedFourThree[] = { '\0', '\0', '\0', '\0' }; + + unsigned char badTrailFour[] = { '\0', '\0', '\0', '\0', '\0' }; + + for (uint8_t lead : IntegerRange(0b1111'0000, 0b1111'1000)) { + truncatedFourOne[0] = lead; + ExpectNotEnoughUnits(truncatedFourOne, 1, 4); - static const unsigned char arbitraryLowSurrogate[] = { 0xED, 0xB7, 0xAF }; - MOZ_RELEASE_ASSERT(!IsValidUtf8(arbitraryLowSurrogate, ArrayLength(arbitraryLowSurrogate))); + truncatedFourTwo[0] = lead; + ExpectNotEnoughUnits(truncatedFourTwo, 2, 4); + + truncatedFourThree[0] = lead; + ExpectNotEnoughUnits(truncatedFourThree, 3, 4); + + badTrailFour[0] = lead; + badTrailFour[2] = badTrailFour[3] = 0b1011'1111; // test for overreads + for (uint8_t second : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailFour[1] = second; + ExpectBadTrailingUnit(badTrailFour, 2); + } + { + uint8_t second = 0b1100'0000; + do { + badTrailFour[1] = second; + ExpectBadTrailingUnit(badTrailFour, 2); + if (second == 0b1111'1111) { + break; + } + + second++; + } while (true); + } + + badTrailFour[1] = badTrailFour[3] = 0b1011'1111; // test for overreads + for (uint8_t third : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailFour[2] = third; + ExpectBadTrailingUnit(badTrailFour, 3); + } + { + uint8_t third = 0b1100'0000; + do { + badTrailFour[2] = third; + ExpectBadTrailingUnit(badTrailFour, 3); + if (third == 0b1111'1111) { + break; + } + + third++; + } while (true); + } + + badTrailFour[2] = 0b1011'1111; + for (uint8_t fourth : IntegerRange(0b0000'0000, 0b1000'0000)) { + badTrailFour[3] = fourth; + ExpectBadTrailingUnit(badTrailFour, 4); + } + { + uint8_t fourth = 0b1100'0000; + do { + badTrailFour[3] = fourth; + ExpectBadTrailingUnit(badTrailFour, 4); + if (fourth == 0b1111'1111) { + break; + } + + fourth++; + } while (true); + } + } +} + +static void +TestBadSurrogate() +{ + // These tests are actually exhaustive. + + ExpectValidCodePoint("\xED\x9F\xBF", 0xD7FF); // last before surrogates + ExpectValidCodePoint("\xEE\x80\x80", 0xE000); // first after surrogates + + // First invalid surrogate encoding is { 0xED, 0xA0, 0x80 }. Last invalid + // surrogate encoding is { 0xED, 0xBF, 0xBF }. + + char badSurrogate[] = { '\xED', '\0', '\0', '\0' }; + + for (char32_t c = 0xD800; c < 0xE000; c++) { + badSurrogate[1] = 0b1000'0000 ^ ((c & 0b1111'1100'0000) >> 6); + badSurrogate[2] = 0b1000'0000 ^ ((c & 0b0000'0011'1111)); + + ExpectBadCodePoint(badSurrogate, c, 3); + } +} + +static void +TestBadTooBig() +{ + // These tests are actually exhaustive. + + ExpectValidCodePoint("\xF4\x8F\xBF\xBF", 0x10'FFFF); // last code point + + // Four-byte code points are + // + // 0b1111'0xxx 0b10xx'xxxx 0b10xx'xxxx 0b10xx'xxxx + // + // with 3 + 6 + 6 + 6 == 21 unconstrained bytes, so the structurally + // representable limit (exclusive) is 2**21 - 1 == 2097152. + + char tooLargeCodePoint[] = { '\0', '\0', '\0', '\0', '\0' }; + + for (char32_t c = 0x11'0000; c < (1 << 21); c++) { + tooLargeCodePoint[0] = 0b1111'0000 ^ ((c & 0b1'1100'0000'0000'0000'0000) >> 18); + tooLargeCodePoint[1] = 0b1000'0000 ^ ((c & 0b0'0011'1111'0000'0000'0000) >> 12); + tooLargeCodePoint[2] = 0b1000'0000 ^ ((c & 0b0'0000'0000'1111'1100'0000) >> 6); + tooLargeCodePoint[3] = 0b1000'0000 ^ ((c & 0b0'0000'0000'0000'0011'1111)); - static const unsigned char greatestSurrogate[] = { 0xED, 0xBF, 0xBF }; - MOZ_RELEASE_ASSERT(!IsValidUtf8(greatestSurrogate, ArrayLength(greatestSurrogate))); + ExpectBadCodePoint(tooLargeCodePoint, c, 4); + } +} + +static void +TestBadCodePoint() +{ + TestBadSurrogate(); + TestBadTooBig(); +} + +static void +TestNotShortestForm() +{ + { + // One-byte in two-byte. + + char oneInTwo[] = { '\0', '\0', '\0' }; + + for (char32_t c = '\0'; c < 0x80; c++) { + oneInTwo[0] = 0b1100'0000 ^ ((c & 0b0111'1100'0000) >> 6); + oneInTwo[1] = 0b1000'0000 ^ ((c & 0b0000'0011'1111)); + + ExpectNotShortestForm(oneInTwo, c, 2); + } + + // One-byte in three-byte. + + char oneInThree[] = { '\0', '\0', '\0', '\0' }; + + for (char32_t c = '\0'; c < 0x80; c++) { + oneInThree[0] = 0b1110'0000 ^ ((c & 0b1111'0000'0000'0000) >> 12); + oneInThree[1] = 0b1000'0000 ^ ((c & 0b0000'1111'1100'0000) >> 6); + oneInThree[2] = 0b1000'0000 ^ ((c & 0b0000'0000'0011'1111)); + + ExpectNotShortestForm(oneInThree, c, 3); + } + + // One-byte in four-byte. + + char oneInFour[] = { '\0', '\0', '\0', '\0', '\0' }; + + for (char32_t c = '\0'; c < 0x80; c++) { + oneInFour[0] = 0b1111'0000 ^ ((c & 0b1'1100'0000'0000'0000'0000) >> 18); + oneInFour[1] = 0b1000'0000 ^ ((c & 0b0'0011'1111'0000'0000'0000) >> 12); + oneInFour[2] = 0b1000'0000 ^ ((c & 0b0'0000'0000'1111'1100'0000) >> 6); + oneInFour[3] = 0b1000'0000 ^ ((c & 0b0'0000'0000'0000'0011'1111)); + + ExpectNotShortestForm(oneInFour, c, 4); + } + } + + { + // Two-byte in three-byte. - static const unsigned char justAfterSurrogates[] = { 0xEE, 0x80, 0x80 }; - MOZ_RELEASE_ASSERT(IsValidUtf8(justAfterSurrogates, ArrayLength(justAfterSurrogates))); + char twoInThree[] = { '\0', '\0', '\0', '\0' }; + + for (char32_t c = 0x80; c < 0x800; c++) { + twoInThree[0] = 0b1110'0000 ^ ((c & 0b1111'0000'0000'0000) >> 12); + twoInThree[1] = 0b1000'0000 ^ ((c & 0b0000'1111'1100'0000) >> 6); + twoInThree[2] = 0b1000'0000 ^ ((c & 0b0000'0000'0011'1111)); + + ExpectNotShortestForm(twoInThree, c, 3); + } + + // Two-byte in four-byte. + + char twoInFour[] = { '\0', '\0', '\0', '\0', '\0' }; + + for (char32_t c = 0x80; c < 0x800; c++) { + twoInFour[0] = 0b1111'0000 ^ ((c & 0b1'1100'0000'0000'0000'0000) >> 18); + twoInFour[1] = 0b1000'0000 ^ ((c & 0b0'0011'1111'0000'0000'0000) >> 12); + twoInFour[2] = 0b1000'0000 ^ ((c & 0b0'0000'0000'1111'1100'0000) >> 6); + twoInFour[3] = 0b1000'0000 ^ ((c & 0b0'0000'0000'0000'0011'1111)); + + ExpectNotShortestForm(twoInFour, c, 4); + } + } + + { + // Three-byte in four-byte. + + char threeInFour[] = { '\0', '\0', '\0', '\0', '\0' }; + + for (char32_t c = 0x800; c < 0x1'0000; c++) { + threeInFour[0] = 0b1111'0000 ^ ((c & 0b1'1100'0000'0000'0000'0000) >> 18); + threeInFour[1] = 0b1000'0000 ^ ((c & 0b0'0011'1111'0000'0000'0000) >> 12); + threeInFour[2] = 0b1000'0000 ^ ((c & 0b0'0000'0000'1111'1100'0000) >> 6); + threeInFour[3] = 0b1000'0000 ^ ((c & 0b0'0000'0000'0000'0011'1111)); + + ExpectNotShortestForm(threeInFour, c, 4); + } + } +} + +static void +TestDecodeOneInvalidUtf8CodePoint() +{ + TestDecodeBadLeadUnit(); + TestTooFewOrBadTrailingUnits(); + TestBadCodePoint(); + TestNotShortestForm(); +} + +static void +TestDecodeOneUtf8CodePoint() +{ + TestDecodeOneValidUtf8CodePoint(); + TestDecodeOneInvalidUtf8CodePoint(); } int main() { TestUtf8Unit(); TestIsValidUtf8(); + TestDecodeOneUtf8CodePoint(); return 0; }
--- a/modules/libpref/init/StaticPrefList.h +++ b/modules/libpref/init/StaticPrefList.h @@ -1136,27 +1136,17 @@ VARCACHE_PREF( //--------------------------------------------------------------------------- // Preferences prefs //--------------------------------------------------------------------------- PREF("preferences.allow.omt-write", bool, true) //--------------------------------------------------------------------------- -// View source prefs -//--------------------------------------------------------------------------- - -VARCACHE_PREF( - "view_source.editor.external", - view_source_editor_external, - bool, false -) - -//--------------------------------------------------------------------------- -// Anti-Tracking prefs +// Privacy prefs //--------------------------------------------------------------------------- VARCACHE_PREF( "privacy.restrict3rdpartystorage.enabled", privacy_restrict3rdpartystorage_enabled, RelaxedAtomicBool, false ) @@ -1169,12 +1159,62 @@ VARCACHE_PREF( // Anti-tracking permission expiration VARCACHE_PREF( "privacy.restrict3rdpartystorage.expiration", privacy_restrict3rdpartystorage_expiration, uint32_t, 2592000 // 30 days (in seconds) ) //--------------------------------------------------------------------------- +// Security prefs +//--------------------------------------------------------------------------- + +VARCACHE_PREF( + "security.csp.enable", + security_csp_enable, + bool, true +) + +VARCACHE_PREF( + "security.csp.experimentalEnabled", + security_csp_experimentalEnabled, + bool, false +) + +VARCACHE_PREF( + "security.csp.enableStrictDynamic", + security_csp_enableStrictDynamic, + bool, true +) + +#ifdef NIGHTLY_BUILD +# define PREF_VALUE true +#else +# define PREF_VALUE false +#endif +VARCACHE_PREF( + "security.csp.enable_violation_events", + security_csp_enable_violation_events, + bool, PREF_VALUE +) +#undef PREF_VALUE + +VARCACHE_PREF( + "security.csp.reporting.script-sample.max-length", + security_csp_reporting_script_sample_max_length, + int32_t, 40 +) + +//--------------------------------------------------------------------------- +// View source prefs +//--------------------------------------------------------------------------- + +VARCACHE_PREF( + "view_source.editor.external", + view_source_editor_external, + bool, false +) + +//--------------------------------------------------------------------------- // End of prefs //--------------------------------------------------------------------------- // clang-format on
--- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -2522,31 +2522,21 @@ pref("font.name-list.monospace.x-math", pref("font.blacklist.underline_offset", "FangSong,Gulim,GulimChe,MingLiU,MingLiU-ExtB,MingLiU_HKSCS,MingLiU-HKSCS-ExtB,MS Gothic,MS Mincho,MS PGothic,MS PMincho,MS UI Gothic,PMingLiU,PMingLiU-ExtB,SimHei,SimSun,SimSun-ExtB,Hei,Kai,Apple LiGothic,Apple LiSung,Osaka"); pref("security.directory", ""); // security-sensitive dialogs should delay button enabling. In milliseconds. pref("security.dialog_enable_delay", 1000); pref("security.notification_enable_delay", 500); -pref("security.csp.enable", true); -pref("security.csp.experimentalEnabled", false); -pref("security.csp.enableStrictDynamic", true); - #if defined(DEBUG) && !defined(ANDROID) // about:welcome has been added until Bug 1448359 is fixed at which time home, newtab, and welcome will all be removed. pref("csp.content_privileged_about_uris_without_csp", "blank,home,newtab,printpreview,srcdoc,welcome"); #endif -#ifdef NIGHTLY_BUILD -pref("security.csp.enable_violation_events", true); -#else -pref("security.csp.enable_violation_events", false); -#endif - // Default Content Security Policy to apply to signed contents. pref("security.signed_content.CSP.default", "script-src 'self'; style-src 'self'"); // Mixed content blocking pref("security.mixed_content.block_active_content", false); pref("security.mixed_content.block_display_content", false); // Upgrade mixed display content before it's blocked
--- a/parser/html/nsHtml5TreeOpExecutor.cpp +++ b/parser/html/nsHtml5TreeOpExecutor.cpp @@ -1130,17 +1130,17 @@ nsHtml5TreeOpExecutor::SetSpeculationRef if (policy != mozilla::net::RP_Unset) { SetSpeculationReferrerPolicy(policy); } } void nsHtml5TreeOpExecutor::AddSpeculationCSP(const nsAString& aCSP) { - if (!CSPService::sCSPEnabled) { + if (!StaticPrefs::security_csp_enable()) { return; } NS_ASSERTION(NS_IsMainThread(), "Wrong thread!"); nsIPrincipal* principal = mDocument->NodePrincipal(); nsCOMPtr<nsIContentSecurityPolicy> preloadCsp; nsresult rv = principal->EnsurePreloadCSP(mDocument, getter_AddRefs(preloadCsp));
--- a/taskcluster/ci/build/linux.yml +++ b/taskcluster/ci/build/linux.yml @@ -995,17 +995,17 @@ linux64-ccov/opt: - builds/releng_base_firefox.py - builds/releng_base_linux_64_builds.py script: "mozharness/scripts/fx_desktop_build.py" secrets: true custom-build-variant-cfg: code-coverage-opt tooltool-downloads: public need-xvfb: true toolchains: - - linux64-clang + - linux64-clang-7 - linux64-rust - linux64-gcc - linux64-sccache linux64-add-on-devel/opt: description: "Linux64 add-on-devel" index: product: firefox
--- a/testing/marionette/jar.mn +++ b/testing/marionette/jar.mn @@ -41,9 +41,10 @@ marionette.jar: #ifdef ENABLE_TESTS content/test2.xul (chrome/test2.xul) content/test_anonymous_content.xul (chrome/test_anonymous_content.xul) content/test_dialog.dtd (chrome/test_dialog.dtd) content/test_dialog.properties (chrome/test_dialog.properties) content/test_dialog.xul (chrome/test_dialog.xul) content/test_nested_iframe.xul (chrome/test_nested_iframe.xul) content/test.xul (chrome/test.xul) + content/PerTestCoverageUtils.jsm (../../tools/code-coverage/PerTestCoverageUtils.jsm) #endif
--- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -126,16 +126,18 @@ class MochitestFormatter(TbplFormatter): """ log_num = 0 def __init__(self): super(MochitestFormatter, self).__init__() def __call__(self, data): output = super(MochitestFormatter, self).__call__(data) + if not output: + return None log_level = data.get('level', 'info').upper() if 'js_source' in data or log_level == 'ERROR': data.pop('js_source', None) output = '%d %s %s' % ( MochitestFormatter.log_num, log_level, output) MochitestFormatter.log_num += 1
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -199,36 +199,40 @@ class CodeCoverageMixin(SingleTestMixin) self.suites[suite] = [] if test not in self.suites[suite]: self.suites[suite].append(test) @property def coverage_args(self): return [] - def set_coverage_env(self, env): + def set_coverage_env(self, env, is_baseline_test=False): # Set the GCOV directory. - gcov_dir = tempfile.mkdtemp() - env['GCOV_PREFIX'] = gcov_dir + self.gcov_dir = tempfile.mkdtemp() + env['GCOV_PREFIX'] = self.gcov_dir + + # Set the GCOV directory where counters will be dumped in per-test mode. + # Resetting/dumping is only available on Linux for the time being + # (https://bugzilla.mozilla.org/show_bug.cgi?id=1471576). + if self.per_test_coverage and not is_baseline_test and self._is_linux(): + env['GCOV_RESULTS_DIR'] = tempfile.mkdtemp() # Set JSVM directory. - jsvm_dir = tempfile.mkdtemp() - env['JS_CODE_COVERAGE_OUTPUT_DIR'] = jsvm_dir - - return (gcov_dir, jsvm_dir) + self.jsvm_dir = tempfile.mkdtemp() + env['JS_CODE_COVERAGE_OUTPUT_DIR'] = self.jsvm_dir @PreScriptAction('run-tests') def _set_gcov_prefix(self, action): if not self.code_coverage_enabled: return if self.per_test_coverage: return - self.gcov_dir, self.jsvm_dir = self.set_coverage_env(os.environ) + self.set_coverage_env(os.environ) def parse_coverage_artifacts(self, gcov_dir, jsvm_dir, merge=False, output_format='lcov', filter_covered=False): jsvm_output_file = 'jsvm_lcov_output.info' @@ -281,30 +285,37 @@ class CodeCoverageMixin(SingleTestMixin) shutil.rmtree(jsvm_dir) if merge: os.remove(jsvm_output_file) return grcov_output_file else: return grcov_output_file, jsvm_output_file - def add_per_test_coverage_report(self, gcov_dir, jsvm_dir, suite, test): + def add_per_test_coverage_report(self, env, suite, test): + gcov_dir = env['GCOV_RESULTS_DIR'] if 'GCOV_RESULTS_DIR' in env else self.gcov_dir + grcov_file = self.parse_coverage_artifacts( - gcov_dir, jsvm_dir, merge=True, output_format='coveralls', + gcov_dir, self.jsvm_dir, merge=True, output_format='coveralls', filter_covered=True, ) report_file = str(uuid.uuid4()) + '.json' shutil.move(grcov_file, report_file) if suite not in self.per_test_reports: self.per_test_reports[suite] = {} assert test not in self.per_test_reports[suite] self.per_test_reports[suite][test] = report_file + if 'GCOV_RESULTS_DIR' in env: + # In this case, parse_coverage_artifacts has removed GCOV_RESULTS_DIR + # so we need to remove GCOV_PREFIX. + shutil.rmtree(self.gcov_dir) + def is_covered(self, sf): # For C/C++ source files, we can consider a file as being uncovered # when all its source lines are uncovered. all_lines_uncovered = all(c is None or c == 0 for c in sf['coverage']) if all_lines_uncovered: return False # For JavaScript files, we can't do the same, as the top-level is always
--- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -10,17 +10,16 @@ author: Jordan Lund """ import os import re import sys import copy import shutil -import tempfile import glob import imp from datetime import datetime, timedelta # load modules from parent dir sys.path.insert(1, os.path.dirname(sys.path[0])) @@ -884,38 +883,28 @@ class DesktopUnittest(TestingMixin, Merc "were executed.<br/>") executed_too_many_tests = True executed_tests = executed_tests + 1 final_cmd = copy.copy(cmd) final_cmd.extend(per_test_args) + final_env = copy.copy(env) + if self.per_test_coverage: - gcov_dir, jsvm_dir = self.set_coverage_env(env) - # Per-test reset/dump is only supported on Linux for the time being. - if not is_baseline_test and \ - self._is_linux(): - env['GCOV_RESULTS_DIR'] = tempfile.mkdtemp() + self.set_coverage_env(final_env) return_code = self.run_command(final_cmd, cwd=dirs['abs_work_dir'], output_timeout=cmd_timeout, output_parser=parser, - env=env) + env=final_env) if self.per_test_coverage: - self.add_per_test_coverage_report( - env['GCOV_RESULTS_DIR'] if 'GCOV_RESULTS_DIR' in env else gcov_dir, - jsvm_dir, - suite, - per_test_args[-1] - ) - if 'GCOV_RESULTS_DIR' in env: - shutil.rmtree(gcov_dir) - del env['GCOV_RESULTS_DIR'] + self.add_per_test_coverage_report(final_env, suite, per_test_args[-1]) # mochitest, reftest, and xpcshell suites do not return # appropriate return codes. Therefore, we must parse the output # to determine what the tbpl_status and worst_log_level must # be. We do this by: # 1) checking to see if our mozharness script ran into any # errors itself with 'num_errors' <- OutputParser # 2) if num_errors is 0 then we look in the subclassed 'parser'
--- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -372,27 +372,29 @@ class WebPlatformTest(TestingMixin, Merc "were executed.<br/>") executed_too_many_tests = True executed_tests = executed_tests + 1 cmd = self._query_cmd(test_types) cmd.extend(per_test_args) + final_env = copy.copy(env) + if self.per_test_coverage: - gcov_dir, jsvm_dir = self.set_coverage_env(env) + self.set_coverage_env(final_env, is_baseline_test) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, - env=env) + env=final_env) if self.per_test_coverage: - self.add_per_test_coverage_report(gcov_dir, jsvm_dir, suite, per_test_args[-1]) + self.add_per_test_coverage_report(final_env, suite, per_test_args[-1]) tbpl_status, log_level, summary = parser.evaluate_parser(return_code, previous_summary=summary) self.record_status(tbpl_status, level=log_level) if len(per_test_args) > 0: self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
--- a/testing/web-platform/meta/css/CSS2/backgrounds/background-position-001.xht.ini +++ b/testing/web-platform/meta/css/CSS2/backgrounds/background-position-001.xht.ini @@ -1,3 +1,3 @@ [background-position-001.xht] expected: - if os == "linux": FAIL + if os == "linux" and not webrender: FAIL
--- a/testing/web-platform/meta/css/CSS2/backgrounds/background-position-002.xht.ini +++ b/testing/web-platform/meta/css/CSS2/backgrounds/background-position-002.xht.ini @@ -1,3 +1,3 @@ [background-position-002.xht] expected: - if os == "linux": FAIL + if os == "linux" and not webrender: FAIL
--- a/testing/web-platform/meta/html/infrastructure/urls/resolving-urls/query-encoding/navigation.sub.html.ini +++ b/testing/web-platform/meta/html/infrastructure/urls/resolving-urls/query-encoding/navigation.sub.html.ini @@ -1,15 +1,14 @@ [navigation.sub.html?encoding=x-cp1251] expected: if not debug and not webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): TIMEOUT - if not debug and not webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT + if not debug and not asan and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT if not debug and not webrender and e10s and (os == "win") and (version == "6.1.7601") and (processor == "x86") and (bits == 32): TIMEOUT if not debug and not webrender and e10s and (os == "win") and (version == "10.0.15063") and (processor == "x86_64") and (bits == 64): TIMEOUT - if not debug and webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT if not debug and not webrender and e10s and (os == "mac") and (version == "OS X 10.10.5") and (processor == "x86_64") and (bits == 64): TIMEOUT [follow hyperlink <a href>] expected: FAIL [follow hyperlink <area href>] expected: FAIL [follow hyperlink <link href>] @@ -20,19 +19,18 @@ [hyperlink auditing <area ping>] expected: TIMEOUT [navigation.sub.html?encoding=utf8] expected: if not debug and not webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): TIMEOUT - if not debug and not webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT + if not debug and not asan and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT if not debug and not webrender and e10s and (os == "win") and (version == "6.1.7601") and (processor == "x86") and (bits == 32): TIMEOUT if not debug and not webrender and e10s and (os == "win") and (version == "10.0.15063") and (processor == "x86_64") and (bits == 64): TIMEOUT - if not debug and webrender and e10s and (os == "linux") and (version == "Ubuntu 16.04") and (processor == "x86_64") and (bits == 64): TIMEOUT if not debug and not webrender and e10s and (os == "mac") and (version == "OS X 10.10.5") and (processor == "x86_64") and (bits == 64): TIMEOUT [hyperlink auditing <a ping>] expected: TIMEOUT [hyperlink auditing <area ping>] expected: TIMEOUT
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py @@ -24,17 +24,18 @@ from .protocol import (AssertsProtocolPa BaseProtocolPart, TestharnessProtocolPart, PrefsProtocolPart, Protocol, StorageProtocolPart, SelectorProtocolPart, ClickProtocolPart, SendKeysProtocolPart, - TestDriverProtocolPart) + TestDriverProtocolPart, + CoverageProtocolPart) from ..testrunner import Stop from ..webdriver_server import GeckoDriverServer def do_delayed_imports(): global errors, marionette # Marionette client used to be called marionette, recently it changed @@ -366,26 +367,74 @@ class MarionetteTestDriverProtocolPart(T "type": "testdriver-%s" % str(message_type), "status": str(status) } if message: obj["message"] = str(message) self.parent.base.execute_script("window.postMessage(%s, '*')" % json.dumps(obj)) +class MarionetteCoverageProtocolPart(CoverageProtocolPart): + def setup(self): + self.marionette = self.parent.marionette + script = """ + ChromeUtils.import("chrome://marionette/content/PerTestCoverageUtils.jsm"); + return PerTestCoverageUtils.enabled; + """ + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + self.is_enabled = self.marionette.execute_script(script) + + def reset(self): + script = """ + var callback = arguments[arguments.length - 1]; + + ChromeUtils.import("chrome://marionette/content/PerTestCoverageUtils.jsm"); + PerTestCoverageUtils.beforeTest().then(callback, callback); + """ + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + try: + error = self.marionette.execute_async_script(script) + if error is not None: + raise Exception('Failure while resetting counters: %s' % json.dumps(error)) + except (errors.MarionetteException, socket.error): + # This usually happens if the process crashed + pass + + def dump(self): + if len(self.marionette.window_handles): + handle = self.marionette.window_handles[0] + self.marionette.switch_to_window(handle) + + script = """ + var callback = arguments[arguments.length - 1]; + + ChromeUtils.import("chrome://marionette/content/PerTestCoverageUtils.jsm"); + PerTestCoverageUtils.afterTest().then(callback, callback); + """ + with self.marionette.using_context(self.marionette.CONTEXT_CHROME): + try: + error = self.marionette.execute_async_script(script) + if error is not None: + raise Exception('Failure while dumping counters: %s' % json.dumps(error)) + except (errors.MarionetteException, socket.error): + # This usually happens if the process crashed + pass + + class MarionetteProtocol(Protocol): implements = [MarionetteBaseProtocolPart, MarionetteTestharnessProtocolPart, MarionettePrefsProtocolPart, MarionetteStorageProtocolPart, MarionetteSelectorProtocolPart, MarionetteClickProtocolPart, MarionetteSendKeysProtocolPart, MarionetteTestDriverProtocolPart, - MarionetteAssertsProtocolPart] + MarionetteAssertsProtocolPart, + MarionetteCoverageProtocolPart] def __init__(self, executor, browser, capabilities=None, timeout_multiplier=1, e10s=True): do_delayed_imports() super(MarionetteProtocol, self).__init__(executor, browser) self.marionette = None self.marionette_port = browser.marionette_port self.capabilities = capabilities @@ -594,16 +643,19 @@ class MarionetteTestharnessExecutor(Test protocol.base.execute_script("if (window.win) {window.win.close()}") parent_window = protocol.testharness.close_old_windows(protocol) if timeout is not None: timeout_ms = str(timeout * 1000) else: timeout_ms = "null" + if self.protocol.coverage.is_enabled: + self.protocol.coverage.reset() + format_map = {"abs_url": url, "url": strip_server(url), "window_id": self.window_id, "timeout_multiplier": self.timeout_multiplier, "timeout": timeout_ms, "explicit_timeout": timeout is None} script = self.script % format_map @@ -616,16 +668,20 @@ class MarionetteTestharnessExecutor(Test result = protocol.base.execute_script( self.script_resume % format_map, async=True) if result is None: # This can happen if we get an content process crash return None done, rv = handler(result) if done: break + + if self.protocol.coverage.is_enabled: + self.protocol.coverage.dump() + return rv class MarionetteRefTestExecutor(RefTestExecutor): def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None, close_after_done=True, debug_info=None, reftest_internal=False, reftest_screenshot="unexpected", @@ -684,18 +740,24 @@ class MarionetteRefTestExecutor(RefTestE self.protocol.marionette.window_handles[-1]) self.has_window = False if not self.has_window: self.protocol.base.execute_script(self.script) self.protocol.base.set_window(self.protocol.marionette.window_handles[-1]) self.has_window = True + if self.protocol.coverage.is_enabled: + self.protocol.coverage.reset() + result = self.implementation.run_test(test) + if self.protocol.coverage.is_enabled: + self.protocol.coverage.dump() + if self.debug: assertion_count = self.protocol.asserts.get() if "extra" not in result: result["extra"] = {} result["extra"]["assertion_count"] = assertion_count return self.convert_result(test, result)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/protocol.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/protocol.py @@ -298,8 +298,25 @@ class AssertsProtocolPart(ProtocolPart): __metaclass__ = ABCMeta name = "asserts" @abstractmethod def get(self): """Get a count of assertions since the last browser start""" pass + + +class CoverageProtocolPart(ProtocolPart): + """Protocol part for collecting per-test coverage data.""" + __metaclass__ = ABCMeta + + name = "coverage" + + @abstractmethod + def reset(self): + """Reset coverage counters""" + pass + + @abstractmethod + def dump(self): + """Dump coverage counters""" + pass
--- a/toolkit/components/antitracking/AntiTrackingCommon.cpp +++ b/toolkit/components/antitracking/AntiTrackingCommon.cpp @@ -2,16 +2,18 @@ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AntiTrackingCommon.h" #include "mozilla/dom/ContentChild.h" +#include "mozilla/ipc/MessageChannel.h" +#include "mozilla/AbstractThread.h" #include "mozilla/StaticPrefs.h" #include "mozIThirdPartyUtil.h" #include "nsContentUtils.h" #include "nsGlobalWindowInner.h" #include "nsIPermissionManager.h" #include "nsIPrincipal.h" #include "nsIURI.h" #include "nsPIDOMWindow.h" @@ -66,99 +68,115 @@ CreatePermissionKey(const nsCString& aTr aPermissionKey = nsPrintfCString(ANTITRACKING_PERM_KEY "^%s^%s", aTrackingOrigin.get(), aGrantedOrigin.get()); } } // anonymous -/* static */ void +/* static */ RefPtr<AntiTrackingCommon::StorageAccessGrantPromise> AntiTrackingCommon::AddFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin, nsPIDOMWindowInner* aParentWindow) { MOZ_ASSERT(aParentWindow); if (!StaticPrefs::privacy_restrict3rdpartystorage_enabled()) { - return; + return StorageAccessGrantPromise::CreateAndResolve(true, __func__); } nsCOMPtr<nsIPrincipal> topLevelStoragePrincipal; nsAutoCString trackingOrigin; nsGlobalWindowInner* parentWindow = nsGlobalWindowInner::Cast(aParentWindow); nsGlobalWindowOuter* outerParentWindow = nsGlobalWindowOuter::Cast(parentWindow->GetOuterWindow()); if (NS_WARN_IF(!outerParentWindow)) { - return; + return StorageAccessGrantPromise::CreateAndReject(false, __func__); } // We are a first party resource. if (outerParentWindow->IsTopLevelWindow()) { CopyUTF16toUTF8(aOrigin, trackingOrigin); topLevelStoragePrincipal = parentWindow->GetPrincipal(); if (NS_WARN_IF(!topLevelStoragePrincipal)) { - return; + return StorageAccessGrantPromise::CreateAndReject(false, __func__); } // We are a 3rd party source. } else if (!GetParentPrincipalAndTrackingOrigin(parentWindow, getter_AddRefs(topLevelStoragePrincipal), trackingOrigin)) { - return; + return StorageAccessGrantPromise::CreateAndReject(false, __func__); } NS_ConvertUTF16toUTF8 grantedOrigin(aOrigin); if (XRE_IsParentProcess()) { + RefPtr<StorageAccessGrantPromise::Private> p = new StorageAccessGrantPromise::Private(__func__); SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(topLevelStoragePrincipal, trackingOrigin, - grantedOrigin); - return; + grantedOrigin, + [p] (bool success) { + p->Resolve(success, __func__); + }); + return p; } ContentChild* cc = ContentChild::GetSingleton(); MOZ_ASSERT(cc); // This is not really secure, because here we have the content process sending // the request of storing a permission. - Unused << cc->SendFirstPartyStorageAccessGrantedForOrigin(IPC::Principal(topLevelStoragePrincipal), - trackingOrigin, - grantedOrigin); + RefPtr<StorageAccessGrantPromise::Private> p = new StorageAccessGrantPromise::Private(__func__); + cc->SendFirstPartyStorageAccessGrantedForOrigin(IPC::Principal(topLevelStoragePrincipal), + trackingOrigin, + grantedOrigin) + ->Then(GetCurrentThreadSerialEventTarget(), __func__, + [p] (bool success) { + p->Resolve(success, __func__); + }, [p] (ipc::ResponseRejectReason aReason) { + p->Reject(false, __func__); + }); + return p; } /* static */ void AntiTrackingCommon::SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(nsIPrincipal* aParentPrincipal, const nsCString& aTrackingOrigin, - const nsCString& aGrantedOrigin) + const nsCString& aGrantedOrigin, + FirstPartyStorageAccessGrantedForOriginResolver&& aResolver) { MOZ_ASSERT(XRE_IsParentProcess()); if (NS_WARN_IF(!aParentPrincipal)) { // The child process is sending something wrong. Let's ignore it. + aResolver(false); return; } nsCOMPtr<nsIPermissionManager> pm = services::GetPermissionManager(); if (NS_WARN_IF(!pm)) { + aResolver(false); return; } // Remember that this pref is stored in seconds! uint32_t expirationTime = StaticPrefs::privacy_restrict3rdpartystorage_expiration() * 1000; int64_t when = (PR_Now() / PR_USEC_PER_MSEC) + expirationTime; nsAutoCString type; CreatePermissionKey(aTrackingOrigin, aGrantedOrigin, type); nsresult rv = pm->AddFromPrincipal(aParentPrincipal, type.get(), nsIPermissionManager::ALLOW_ACTION, nsIPermissionManager::EXPIRE_TIME, when); Unused << NS_WARN_IF(NS_FAILED(rv)); + aResolver(NS_SUCCEEDED(rv)); } bool AntiTrackingCommon::IsFirstPartyStorageAccessGrantedFor(nsPIDOMWindowInner* a3rdPartyTrackingWindow, nsIURI* aURI) { MOZ_ASSERT(a3rdPartyTrackingWindow); MOZ_ASSERT(aURI);
--- a/toolkit/components/antitracking/AntiTrackingCommon.h +++ b/toolkit/components/antitracking/AntiTrackingCommon.h @@ -3,27 +3,36 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef mozilla_antitrackingservice_h #define mozilla_antitrackingservice_h #include "nsString.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" class nsIHttpChannel; class nsIPrincipal; class nsIURI; class nsPIDOMWindowInner; namespace mozilla { class AntiTrackingCommon final { public: + // Normally we would include PContentParent.h here and use the + // ipc::FirstPartyStorageAccessGrantedForOriginResolver type which maps to + // the same underlying type, but that results in Windows compilation errors, + // so we use the underlying type to avoid the #include here. + typedef std::function<void(const bool&)> + FirstPartyStorageAccessGrantedForOriginResolver; + // This method returns true if the URI has first party storage access when // loaded inside the passed 3rd party context tracking resource window. // If the window is first party context, please use // MaybeIsFirstPartyStorageAccessGrantedFor(); static bool IsFirstPartyStorageAccessGrantedFor(nsPIDOMWindowInner* a3rdPartyTrackingWindow, nsIURI* aURI); @@ -54,23 +63,25 @@ public: // loaded by tracker.com when loaded by example.net. // - aParentWindow is a first party context and a 3rd party resource (probably // becuase of a script) opens a popup and the user interacts with it. We // want to grant the permission for the 3rd party context to have access to // the first party stoage when loaded in aParentWindow. // Ex: example.net import tracker.com/script.js which does opens a popup and // the user interacts with it. tracker.com is allowed when loaded by // example.net. - static void + typedef MozPromise<bool, bool, false> StorageAccessGrantPromise; + static MOZ_MUST_USE RefPtr<StorageAccessGrantPromise> AddFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin, nsPIDOMWindowInner* aParentWindow); // For IPC only. static void SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(nsIPrincipal* aPrincipal, const nsCString& aParentOrigin, - const nsCString& aGrantedOrigin); + const nsCString& aGrantedOrigin, + FirstPartyStorageAccessGrantedForOriginResolver&& aResolver); }; } // namespace mozilla #endif // mozilla_antitrackingservice_h
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm +++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm @@ -1119,41 +1119,33 @@ EnvironmentCache.prototype = { QueryInterface: ChromeUtils.generateQI([Ci.nsISupportsWeakReference]), /** * Start watching the preferences. */ _startWatchingPrefs() { this._log.trace("_startWatchingPrefs - " + this._watchedPrefs); - for (let [pref, options] of this._watchedPrefs) { - if (!("requiresRestart" in options) || !options.requiresRestart) { - Services.prefs.addObserver(pref, this, true); - } - } + Services.prefs.addObserver("", this, true); }, _onPrefChanged(aData) { this._log.trace("_onPrefChanged"); let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); this._currentEnvironment.settings.userPrefs[aData] = this._getPrefValue(aData, this._watchedPrefs.get(aData).what); this._onEnvironmentChange("pref-changed", oldEnvironment); }, /** * Do not receive any more change notifications for the preferences. */ _stopWatchingPrefs() { this._log.trace("_stopWatchingPrefs"); - for (let [pref, options] of this._watchedPrefs) { - if (!("requiresRestart" in options) || !options.requiresRestart) { - Services.prefs.removeObserver(pref, this); - } - } + Services.prefs.removeObserver("", this); }, _addObservers() { // Watch the search engine change and service topics. Services.obs.addObserver(this, SESSIONSTORE_WINDOWS_RESTORED_TOPIC); Services.obs.addObserver(this, COMPOSITOR_CREATED_TOPIC); Services.obs.addObserver(this, COMPOSITOR_PROCESS_ABORTED_TOPIC); Services.obs.addObserver(this, DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC); @@ -1215,17 +1207,18 @@ EnvironmentCache.prototype = { // Make sure to initialize the search service once we've done restoring // the windows, so that we don't risk loosing search data. Services.search.init(); // The default browser check could take some time, so just call it after // the session was restored. this._updateDefaultBrowser(); break; case PREF_CHANGED_TOPIC: - if (this._watchedPrefs.has(aData)) { + let options = this._watchedPrefs.get(aData); + if (options && !options.requiresRestart) { this._onPrefChanged(aData); } break; } }, /** * Get the default search engine.
--- a/toolkit/content/browser-content.js +++ b/toolkit/content/browser-content.js @@ -35,16 +35,19 @@ XPCOMUtils.defineLazyProxy(this, "PopupB let tmp = {}; ChromeUtils.import("resource://gre/modules/PopupBlocking.jsm", tmp); return new tmp.PopupBlocking(global); }); XPCOMUtils.defineLazyProxy(this, "SelectionSourceContent", "resource://gre/modules/SelectionSourceContent.jsm"); +XPCOMUtils.defineLazyProxy(this, "WebChannelContent", + "resource://gre/modules/WebChannelContent.jsm"); + XPCOMUtils.defineLazyProxy(this, "DateTimePickerContent", () => { let tmp = {}; ChromeUtils.import("resource://gre/modules/DateTimePickerContent.jsm", tmp); return new tmp.DateTimePickerContent(this); }); // Lazily load the finder code @@ -280,100 +283,19 @@ var FindBar = { _onMouseup(event) { if (this._findMode != this.FIND_NORMAL) sendAsyncMessage("Findbar:Mouseup"); }, }; FindBar.init(); -let WebChannelMessageToChromeListener = { - // Preference containing the list (space separated) of origins that are - // allowed to send non-string values through a WebChannel, mainly for - // backwards compatability. See bug 1238128 for more information. - URL_WHITELIST_PREF: "webchannel.allowObject.urlWhitelist", - - // Cached list of whitelisted principals, we avoid constructing this if the - // value in `_lastWhitelistValue` hasn't changed since we constructed it last. - _cachedWhitelist: [], - _lastWhitelistValue: "", - - init() { - addEventListener("WebChannelMessageToChrome", e => { - this._onMessageToChrome(e); - }, true, true); - }, - - _getWhitelistedPrincipals() { - let whitelist = Services.prefs.getCharPref(this.URL_WHITELIST_PREF); - if (whitelist != this._lastWhitelistValue) { - let urls = whitelist.split(/\s+/); - this._cachedWhitelist = urls.map(origin => - Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin)); - } - return this._cachedWhitelist; - }, - - _onMessageToChrome(e) { - // If target is window then we want the document principal, otherwise fallback to target itself. - let principal = e.target.nodePrincipal ? e.target.nodePrincipal : e.target.document.nodePrincipal; - - if (e.detail) { - if (typeof e.detail != "string") { - // Check if the principal is one of the ones that's allowed to send - // non-string values for e.detail. They're whitelisted by site origin, - // so we compare on originNoSuffix in order to avoid other origin attributes - // that are not relevant here, such as containers or private browsing. - let objectsAllowed = this._getWhitelistedPrincipals().some(whitelisted => - principal.originNoSuffix == whitelisted.originNoSuffix); - if (!objectsAllowed) { - Cu.reportError("WebChannelMessageToChrome sent with an object from a non-whitelisted principal"); - return; - } - } - sendAsyncMessage("WebChannelMessageToChrome", e.detail, { eventTarget: e.target }, principal); - } else { - Cu.reportError("WebChannel message failed. No message detail."); - } - } -}; - -WebChannelMessageToChromeListener.init(); - -// This should be kept in sync with /browser/base/content.js. -// Add message listener for "WebChannelMessageToContent" messages from chrome scripts. -addMessageListener("WebChannelMessageToContent", function(e) { - if (e.data) { - // e.objects.eventTarget will be defined if sending a response to - // a WebChannelMessageToChrome event. An unsolicited send - // may not have an eventTarget defined, in this case send to the - // main content window. - let eventTarget = e.objects.eventTarget || content; - - // Use nodePrincipal if available, otherwise fallback to document principal. - let targetPrincipal = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget.document.nodePrincipal : eventTarget.nodePrincipal; - - if (e.principal.subsumes(targetPrincipal)) { - // If eventTarget is a window, use it as the targetWindow, otherwise - // find the window that owns the eventTarget. - let targetWindow = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget : eventTarget.ownerGlobal; - - eventTarget.dispatchEvent(new targetWindow.CustomEvent("WebChannelMessageToContent", { - detail: Cu.cloneInto({ - id: e.data.id, - message: e.data.message, - }, targetWindow), - })); - } else { - Cu.reportError("WebChannel message failed. Principal mismatch."); - } - } else { - Cu.reportError("WebChannel message failed. No message data."); - } -}); +addEventListener("WebChannelMessageToChrome", WebChannelContent, + true, true); +addMessageListener("WebChannelMessageToContent", WebChannelContent); var AudioPlaybackListener = { QueryInterface: ChromeUtils.generateQI([Ci.nsIObserver]), init() { Services.obs.addObserver(this, "audio-playback"); addMessageListener("AudioPlayback", this);
copy from toolkit/content/browser-content.js copy to toolkit/modules/WebChannelContent.jsm --- a/toolkit/content/browser-content.js +++ b/toolkit/modules/WebChannelContent.jsm @@ -1,310 +1,50 @@ /* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/* eslint-env mozilla/frame-script */ /* eslint no-unused-vars: ["error", {args: "none"}] */ -/* global sendAsyncMessage */ + +var EXPORTED_SYMBOLS = ["WebChannelContent"]; ChromeUtils.import("resource://gre/modules/Services.jsm"); -ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); -ChromeUtils.defineModuleGetter(this, "AutoCompletePopup", - "resource://gre/modules/AutoCompletePopupContent.jsm"); -ChromeUtils.defineModuleGetter(this, "AutoScrollController", - "resource://gre/modules/AutoScrollController.jsm"); -ChromeUtils.defineModuleGetter(this, "BrowserUtils", - "resource://gre/modules/BrowserUtils.jsm"); -ChromeUtils.defineModuleGetter(this, "SelectContentHelper", - "resource://gre/modules/SelectContentHelper.jsm"); -ChromeUtils.defineModuleGetter(this, "FindContent", - "resource://gre/modules/FindContent.jsm"); -ChromeUtils.defineModuleGetter(this, "PrintingContent", - "resource://gre/modules/PrintingContent.jsm"); -ChromeUtils.defineModuleGetter(this, "RemoteFinder", - "resource://gre/modules/RemoteFinder.jsm"); - -XPCOMUtils.defineLazyServiceGetter(this, "formFill", - "@mozilla.org/satchel/form-fill-controller;1", - "nsIFormFillController"); - -var global = this; - -XPCOMUtils.defineLazyProxy(this, "PopupBlocking", () => { - let tmp = {}; - ChromeUtils.import("resource://gre/modules/PopupBlocking.jsm", tmp); - return new tmp.PopupBlocking(global); -}); - -XPCOMUtils.defineLazyProxy(this, "SelectionSourceContent", - "resource://gre/modules/SelectionSourceContent.jsm"); - -XPCOMUtils.defineLazyProxy(this, "DateTimePickerContent", () => { - let tmp = {}; - ChromeUtils.import("resource://gre/modules/DateTimePickerContent.jsm", tmp); - return new tmp.DateTimePickerContent(this); -}); - - -// Lazily load the finder code -addMessageListener("Finder:Initialize", function() { - let {RemoteFinderListener} = ChromeUtils.import("resource://gre/modules/RemoteFinder.jsm", {}); - new RemoteFinderListener(global); -}); +function getMessageManager(event) { + let window = Cu.getGlobalForObject(event.target); -var AutoScrollListener = { - handleEvent(event) { - if (event.isTrusted & - !event.defaultPrevented && - event.button == 1) { - if (!this._controller) { - this._controller = new AutoScrollController(global); - } - this._controller.handleEvent(event); - } - } -}; -Services.els.addSystemEventListener(global, "mousedown", AutoScrollListener, true); - -addEventListener("MozOpenDateTimePicker", DateTimePickerContent); - -addEventListener("DOMPopupBlocked", PopupBlocking, true); - -var Printing = { - MESSAGES: [ - "Printing:Preview:Enter", - "Printing:Preview:Exit", - "Printing:Preview:Navigate", - "Printing:Preview:ParseDocument", - "Printing:Print", - ], - - init() { - this.MESSAGES.forEach(msgName => addMessageListener(msgName, this)); - addEventListener("PrintingError", this, true); - addEventListener("printPreviewUpdate", this, true); - this.init = null; - }, - - handleEvent(event) { - return PrintingContent.handleEvent(global, event); - }, - - receiveMessage(message) { - return PrintingContent.receiveMessage(global, message); - }, -}; -Printing.init(); - -function SwitchDocumentDirection(aWindow) { - // document.dir can also be "auto", in which case it won't change - if (aWindow.document.dir == "ltr" || aWindow.document.dir == "") { - aWindow.document.dir = "rtl"; - } else if (aWindow.document.dir == "rtl") { - aWindow.document.dir = "ltr"; - } - for (let run = 0; run < aWindow.frames.length; run++) { - SwitchDocumentDirection(aWindow.frames[run]); - } + return window.document.docShell + .QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIContentFrameMessageManager); } -addMessageListener("SwitchDocumentDirection", () => { - SwitchDocumentDirection(content.window); -}); - -var FindBar = { - /* Please keep in sync with toolkit/content/widgets/findbar.xml */ - FIND_NORMAL: 0, - FIND_TYPEAHEAD: 1, - FIND_LINKS: 2, - - _findMode: 0, - - /** - * _findKey and _findModifiers are used to determine whether a keypress - * is a user attempting to use the find shortcut, after which we'll - * route keypresses to the parent until we know the findbar has focus - * there. To do this, we need shortcut data from the parent. - */ - _findKey: null, - _findModifiers: null, - - init() { - addMessageListener("Findbar:UpdateState", this); - Services.els.addSystemEventListener(global, "keypress", this, false); - Services.els.addSystemEventListener(global, "mouseup", this, false); - this._initShortcutData(); - this.init = null; - }, - - receiveMessage(msg) { - switch (msg.name) { - case "Findbar:UpdateState": - this._findMode = msg.data.findMode; - this._quickFindTimeout = msg.data.hasQuickFindTimeout; - if (msg.data.isOpenAndFocused) { - this._keepPassingUntilToldOtherwise = false; - } - break; - case "Findbar:ShortcutData": - // Set us up to never need this again for the lifetime of this process, - // and remove the listener. - Services.cpmm.initialProcessData.findBarShortcutData = msg.data; - Services.cpmm.removeMessageListener("Findbar:ShortcutData", this); - this._initShortcutData(msg.data); - break; - } - }, - - handleEvent(event) { - switch (event.type) { - case "keypress": - this._onKeypress(event); - break; - case "mouseup": - this._onMouseup(event); - break; - } - }, - - /** - * Use initial process data for find key/modifier data if we have it. - * Otherwise, add a listener so we get the data when the parent process has - * it. - */ - _initShortcutData(data = Services.cpmm.initialProcessData.findBarShortcutData) { - if (data) { - this._findKey = data.key; - this._findModifiers = data.modifiers; - } else { - Services.cpmm.addMessageListener("Findbar:ShortcutData", this); - } - }, - - /** - * Check whether this key event will start the findbar in the parent, - * in which case we should pass any further key events to the parent to avoid - * them being lost. - * @param aEvent the key event to check. - */ - _eventMatchesFindShortcut(aEvent) { - let modifiers = this._findModifiers; - if (!modifiers) { - return false; - } - return aEvent.ctrlKey == modifiers.ctrlKey && aEvent.altKey == modifiers.altKey && - aEvent.shiftKey == modifiers.shiftKey && aEvent.metaKey == modifiers.metaKey && - aEvent.key == this._findKey; - }, - - /** - * Returns whether FAYT can be used for the given event in - * the current content state. - */ - _canAndShouldFastFind() { - let should = false; - let can = BrowserUtils.canFastFind(content); - if (can) { - // XXXgijs: why all these shenanigans? Why not use the event's target? - let focusedWindow = {}; - let elt = Services.focus.getFocusedElementForWindow(content, true, focusedWindow); - let win = focusedWindow.value; - should = BrowserUtils.shouldFastFind(elt, win); - } - return { can, should }; - }, - - _onKeypress(event) { - const FAYT_LINKS_KEY = "'"; - const FAYT_TEXT_KEY = "/"; - if (this._eventMatchesFindShortcut(event)) { - this._keepPassingUntilToldOtherwise = true; - } - // Useless keys: - if (event.ctrlKey || event.altKey || event.metaKey || event.defaultPrevented) { - return; - } - - // Check the focused element etc. - let fastFind = this._canAndShouldFastFind(); - - // Can we even use find in this page at all? - if (!fastFind.can) { - return; - } - if (this._keepPassingUntilToldOtherwise) { - this._passKeyToParent(event); - return; - } - if (!fastFind.should) { - return; - } - - let charCode = event.charCode; - // If the find bar is open and quick find is on, send the key to the parent. - if (this._findMode != this.FIND_NORMAL && this._quickFindTimeout) { - if (!charCode) - return; - this._passKeyToParent(event); - } else { - let key = charCode ? String.fromCharCode(charCode) : null; - let manualstartFAYT = (key == FAYT_LINKS_KEY || key == FAYT_TEXT_KEY) && RemoteFinder._manualFAYT; - let autostartFAYT = !manualstartFAYT && RemoteFinder._findAsYouType && key && key != " "; - if (manualstartFAYT || autostartFAYT) { - let mode = (key == FAYT_LINKS_KEY || (autostartFAYT && RemoteFinder._typeAheadLinksOnly)) ? - this.FIND_LINKS : this.FIND_TYPEAHEAD; - // Set _findMode immediately (without waiting for child->parent->child roundtrip) - // to ensure we pass any further keypresses, too. - this._findMode = mode; - this._passKeyToParent(event); - } - } - }, - - _passKeyToParent(event) { - event.preventDefault(); - // These are the properties required to dispatch another 'real' event - // to the findbar in the parent in _dispatchKeypressEvent in findbar.xml . - // If you make changes here, verify that that method can still do its job. - const kRequiredProps = [ - "type", "bubbles", "cancelable", "ctrlKey", "altKey", "shiftKey", - "metaKey", "keyCode", "charCode", - ]; - let fakeEvent = {}; - for (let prop of kRequiredProps) { - fakeEvent[prop] = event[prop]; - } - sendAsyncMessage("Findbar:Keypress", fakeEvent); - }, - - _onMouseup(event) { - if (this._findMode != this.FIND_NORMAL) - sendAsyncMessage("Findbar:Mouseup"); - }, -}; -FindBar.init(); - -let WebChannelMessageToChromeListener = { +var WebChannelContent = { // Preference containing the list (space separated) of origins that are // allowed to send non-string values through a WebChannel, mainly for // backwards compatability. See bug 1238128 for more information. URL_WHITELIST_PREF: "webchannel.allowObject.urlWhitelist", // Cached list of whitelisted principals, we avoid constructing this if the // value in `_lastWhitelistValue` hasn't changed since we constructed it last. _cachedWhitelist: [], _lastWhitelistValue: "", - init() { - addEventListener("WebChannelMessageToChrome", e => { - this._onMessageToChrome(e); - }, true, true); + handleEvent(event) { + if (event.type === "WebChannelMessageToChrome") { + return this._onMessageToChrome(event); + } + return undefined; + }, + + receiveMessage(msg) { + if (msg.name === "WebChannelMessageToContent") { + return this._onMessageToContent(msg); + } + return undefined; }, _getWhitelistedPrincipals() { let whitelist = Services.prefs.getCharPref(this.URL_WHITELIST_PREF); if (whitelist != this._lastWhitelistValue) { let urls = whitelist.split(/\s+/); this._cachedWhitelist = urls.map(origin => Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(origin)); @@ -324,281 +64,47 @@ let WebChannelMessageToChromeListener = // that are not relevant here, such as containers or private browsing. let objectsAllowed = this._getWhitelistedPrincipals().some(whitelisted => principal.originNoSuffix == whitelisted.originNoSuffix); if (!objectsAllowed) { Cu.reportError("WebChannelMessageToChrome sent with an object from a non-whitelisted principal"); return; } } - sendAsyncMessage("WebChannelMessageToChrome", e.detail, { eventTarget: e.target }, principal); + + let mm = getMessageManager(e); + + mm.sendAsyncMessage("WebChannelMessageToChrome", e.detail, { eventTarget: e.target }, principal); } else { Cu.reportError("WebChannel message failed. No message detail."); } - } -}; - -WebChannelMessageToChromeListener.init(); - -// This should be kept in sync with /browser/base/content.js. -// Add message listener for "WebChannelMessageToContent" messages from chrome scripts. -addMessageListener("WebChannelMessageToContent", function(e) { - if (e.data) { - // e.objects.eventTarget will be defined if sending a response to - // a WebChannelMessageToChrome event. An unsolicited send - // may not have an eventTarget defined, in this case send to the - // main content window. - let eventTarget = e.objects.eventTarget || content; - - // Use nodePrincipal if available, otherwise fallback to document principal. - let targetPrincipal = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget.document.nodePrincipal : eventTarget.nodePrincipal; - - if (e.principal.subsumes(targetPrincipal)) { - // If eventTarget is a window, use it as the targetWindow, otherwise - // find the window that owns the eventTarget. - let targetWindow = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget : eventTarget.ownerGlobal; - - eventTarget.dispatchEvent(new targetWindow.CustomEvent("WebChannelMessageToContent", { - detail: Cu.cloneInto({ - id: e.data.id, - message: e.data.message, - }, targetWindow), - })); - } else { - Cu.reportError("WebChannel message failed. Principal mismatch."); - } - } else { - Cu.reportError("WebChannel message failed. No message data."); - } -}); - -var AudioPlaybackListener = { - QueryInterface: ChromeUtils.generateQI([Ci.nsIObserver]), - - init() { - Services.obs.addObserver(this, "audio-playback"); - - addMessageListener("AudioPlayback", this); - addEventListener("unload", () => { - AudioPlaybackListener.uninit(); - }); - this.init = null; - }, - - uninit() { - Services.obs.removeObserver(this, "audio-playback"); - - removeMessageListener("AudioPlayback", this); }, - handleMediaControlMessage(msg) { - let utils = global.content.QueryInterface(Ci.nsIInterfaceRequestor) - .getInterface(Ci.nsIDOMWindowUtils); - let suspendTypes = Ci.nsISuspendedTypes; - switch (msg) { - case "mute": - utils.audioMuted = true; - break; - case "unmute": - utils.audioMuted = false; - break; - case "lostAudioFocus": - utils.mediaSuspend = suspendTypes.SUSPENDED_PAUSE_DISPOSABLE; - break; - case "lostAudioFocusTransiently": - utils.mediaSuspend = suspendTypes.SUSPENDED_PAUSE; - break; - case "gainAudioFocus": - utils.mediaSuspend = suspendTypes.NONE_SUSPENDED; - break; - case "mediaControlPaused": - utils.mediaSuspend = suspendTypes.SUSPENDED_PAUSE_DISPOSABLE; - break; - case "mediaControlStopped": - utils.mediaSuspend = suspendTypes.SUSPENDED_STOP_DISPOSABLE; - break; - case "resumeMedia": - // User has clicked the tab audio indicator to play a delayed - // media. That's clear user intent to play, so gesture activate - // the content document tree so that the block-autoplay logic - // allows the media to autoplay. - content.document.notifyUserGestureActivation(); - utils.mediaSuspend = suspendTypes.NONE_SUSPENDED; - break; - default: - dump("Error : wrong media control msg!\n"); - break; - } - }, + _onMessageToContent(msg) { + if (msg.data) { + // msg.objects.eventTarget will be defined if sending a response to + // a WebChannelMessageToChrome event. An unsolicited send + // may not have an eventTarget defined, in this case send to the + // main content window. + let eventTarget = msg.objects.eventTarget || msg.target.content; + + // Use nodePrincipal if available, otherwise fallback to document principal. + let targetPrincipal = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget.document.nodePrincipal : eventTarget.nodePrincipal; - observe(subject, topic, data) { - if (topic === "audio-playback") { - if (subject && subject.top == global.content) { - let name = "AudioPlayback:"; - if (data === "activeMediaBlockStart") { - name += "ActiveMediaBlockStart"; - } else if (data === "activeMediaBlockStop") { - name += "ActiveMediaBlockStop"; - } else { - name += (data === "active") ? "Start" : "Stop"; - } - sendAsyncMessage(name); + if (msg.principal.subsumes(targetPrincipal)) { + // If eventTarget is a window, use it as the targetWindow, otherwise + // find the window that owns the eventTarget. + let targetWindow = eventTarget instanceof Ci.nsIDOMWindow ? eventTarget : eventTarget.ownerGlobal; + + eventTarget.dispatchEvent(new targetWindow.CustomEvent("WebChannelMessageToContent", { + detail: Cu.cloneInto({ + id: msg.data.id, + message: msg.data.message, + }, targetWindow), + })); + } else { + Cu.reportError("WebChannel message failed. Principal mismatch."); } - } - }, - - receiveMessage(msg) { - if (msg.name == "AudioPlayback") { - this.handleMediaControlMessage(msg.data.type); + } else { + Cu.reportError("WebChannel message failed. No message data."); } }, }; -AudioPlaybackListener.init(); - -var UnselectedTabHoverObserver = { - init() { - addMessageListener("Browser:UnselectedTabHover", this); - addEventListener("UnselectedTabHover:Enable", this); - addEventListener("UnselectedTabHover:Disable", this); - this.init = null; - }, - receiveMessage(message) { - Services.obs.notifyObservers(content.window, "unselected-tab-hover", - message.data.hovered); - }, - handleEvent(event) { - sendAsyncMessage("UnselectedTabHover:Toggle", - { enable: event.type == "UnselectedTabHover:Enable" }); - } -}; -UnselectedTabHoverObserver.init(); - -addMessageListener("Browser:PurgeSessionHistory", function BrowserPurgeHistory() { - let sessionHistory = docShell.QueryInterface(Ci.nsIWebNavigation).sessionHistory; - if (!sessionHistory) { - return; - } - - // place the entry at current index at the end of the history list, so it won't get removed - if (sessionHistory.index < sessionHistory.count - 1) { - let legacy = sessionHistory.legacySHistory; - legacy.QueryInterface(Ci.nsISHistoryInternal); - let indexEntry = legacy.getEntryAtIndex(sessionHistory.index, false); - indexEntry.QueryInterface(Ci.nsISHEntry); - legacy.addEntry(indexEntry, true); - } - - let purge = sessionHistory.count; - if (global.content.location.href != "about:blank") { - --purge; // Don't remove the page the user's staring at from shistory - } - - if (purge > 0) { - sessionHistory.legacySHistory.PurgeHistory(purge); - } -}); - -addMessageListener("ViewSource:GetSelection", SelectionSourceContent); - -addEventListener("MozApplicationManifest", function(e) { - let doc = e.target; - let info = { - uri: doc.documentURI, - characterSet: doc.characterSet, - manifest: doc.documentElement.getAttribute("manifest"), - principal: doc.nodePrincipal, - }; - sendAsyncMessage("MozApplicationManifest", info); -}, false); - -let AutoComplete = { - _connected: false, - - init() { - addEventListener("unload", this, {once: true}); - addEventListener("DOMContentLoaded", this, {once: true}); - // WebExtension browserAction is preloaded and does not receive DCL, wait - // on pageshow so we can hookup the formfill controller. - addEventListener("pageshow", this, {capture: true, once: true}); - - XPCOMUtils.defineLazyProxy(this, "popup", () => new AutoCompletePopup(global), - {QueryInterface: null}); - this.init = null; - }, - - handleEvent(event) { - switch (event.type) { - case "DOMContentLoaded": - case "pageshow": - // We need to wait for a content viewer to be available - // before we can attach our AutoCompletePopup handler, - // since nsFormFillController assumes one will exist - // when we call attachToBrowser. - if (!this._connected) { - formFill.attachToBrowser(docShell, this.popup); - this._connected = true; - } - break; - - case "unload": - if (this._connected) { - formFill.detachFromBrowser(docShell); - this._connected = false; - } - break; - } - }, -}; - -AutoComplete.init(); - -addEventListener("mozshowdropdown", event => { - if (!event.isTrusted) - return; - - if (!SelectContentHelper.open) { - new SelectContentHelper(event.target, {isOpenedViaTouch: false}, this); - } -}); - -addEventListener("mozshowdropdown-sourcetouch", event => { - if (!event.isTrusted) - return; - - if (!SelectContentHelper.open) { - new SelectContentHelper(event.target, {isOpenedViaTouch: true}, this); - } -}); - -let ExtFind = { - init() { - addMessageListener("ext-Finder:CollectResults", this); - addMessageListener("ext-Finder:HighlightResults", this); - addMessageListener("ext-Finder:clearHighlighting", this); - this.init = null; - }, - - _findContent: null, - - async receiveMessage(message) { - if (!this._findContent) { - this._findContent = new FindContent(docShell); - } - - let data; - switch (message.name) { - case "ext-Finder:CollectResults": - this.finderInited = true; - data = await this._findContent.findRanges(message.data); - sendAsyncMessage("ext-Finder:CollectResultsFinished", data); - break; - case "ext-Finder:HighlightResults": - data = this._findContent.highlightResults(message.data); - sendAsyncMessage("ext-Finder:HighlightResultsFinished", data); - break; - case "ext-Finder:clearHighlighting": - this._findContent.highlighter.highlight(false); - break; - } - }, -}; - -ExtFind.init();
--- a/toolkit/modules/moz.build +++ b/toolkit/modules/moz.build @@ -248,16 +248,17 @@ EXTRA_JS_MODULES += [ 'sessionstore/ScrollPosition.jsm', 'ShortcutUtils.jsm', 'Sqlite.jsm', 'Task.jsm', 'Timer.jsm', 'Troubleshoot.jsm', 'UpdateUtils.jsm', 'WebChannel.jsm', + 'WebChannelContent.jsm', 'WindowDraggingUtils.jsm', 'ZipUtils.jsm', ] if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa': EXTRA_JS_MODULES += [ 'PropertyListUtils.jsm', ]
--- a/tools/profiler/gecko/nsProfiler.cpp +++ b/tools/profiler/gecko/nsProfiler.cpp @@ -608,18 +608,18 @@ nsProfiler::StartGathering(double aSince // come in, they will be inserted and end up in the right spot. // FinishGathering() will close the array and the root object. mPendingProfiles = profiles.Length(); RefPtr<nsProfiler> self = this; for (auto profile : profiles) { profile->Then(GetMainThreadSerialEventTarget(), __func__, [self](const mozilla::ipc::Shmem& aResult) { - const nsDependentCString profileString(aResult.get<char>(), - aResult.Size<char>()); + const nsDependentCSubstring profileString(aResult.get<char>(), + aResult.Size<char>() - 1); self->GatheredOOPProfile(profileString); }, [self](ipc::ResponseRejectReason aReason) { self->GatheredOOPProfile(NS_LITERAL_CSTRING("")); }); } if (!mPendingProfiles) { FinishGathering();
--- a/widget/nsClipboardProxy.cpp +++ b/widget/nsClipboardProxy.cpp @@ -94,28 +94,28 @@ nsClipboardProxy::GetData(nsITransferabl mozilla::ipc::Shmem data = item.data().get_Shmem(); if (flavor.EqualsLiteral(kJPEGImageMime) || flavor.EqualsLiteral(kJPGImageMime) || flavor.EqualsLiteral(kPNGImageMime) || flavor.EqualsLiteral(kGIFImageMime)) { nsCOMPtr<nsIInputStream> stream; NS_NewCStringInputStream(getter_AddRefs(stream), - nsDependentCString(data.get<char>(), data.Size<char>())); + nsDependentCSubstring(data.get<char>(), data.Size<char>())); rv = aTransferable->SetTransferData(flavor.get(), stream, sizeof(nsISupports*)); NS_ENSURE_SUCCESS(rv, rv); } else if (flavor.EqualsLiteral(kNativeHTMLMime) || flavor.EqualsLiteral(kRTFMime) || flavor.EqualsLiteral(kCustomTypesMime)) { nsCOMPtr<nsISupportsCString> dataWrapper = do_CreateInstance(NS_SUPPORTS_CSTRING_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); - rv = dataWrapper->SetData(nsDependentCString(data.get<char>(), data.Size<char>())); + rv = dataWrapper->SetData(nsDependentCSubstring(data.get<char>(), data.Size<char>())); NS_ENSURE_SUCCESS(rv, rv); rv = aTransferable->SetTransferData(item.flavor().get(), dataWrapper, data.Size<char>()); NS_ENSURE_SUCCESS(rv, rv); } mozilla::Unused << ContentChild::GetSingleton()->DeallocShmem(data);
--- a/xpcom/base/nsTraceRefcnt.cpp +++ b/xpcom/base/nsTraceRefcnt.cpp @@ -6,17 +6,19 @@ #include "nsTraceRefcnt.h" #include "mozilla/CycleCollectedJSContext.h" #include "mozilla/IntegerPrintfMacros.h" #include "mozilla/Path.h" #include "mozilla/StaticPtr.h" #include "nsXPCOMPrivate.h" #include "nscore.h" +#include "nsClassHashtable.h" #include "nsISupports.h" +#include "nsHashKeys.h" #include "nsTArray.h" #include "nsTHashtable.h" #include "prenv.h" #include "plstr.h" #include "prlink.h" #include "nsCRT.h" #include <math.h> #include "nsHashKeys.h" @@ -47,18 +49,16 @@ #ifdef MOZ_DMD #include "base/process_util.h" #include "nsMemoryInfoDumper.h" #endif //////////////////////////////////////////////////////////////////////////////// -#include "plhash.h" - #include "prthread.h" // We use a spin lock instead of a regular mutex because this lock is usually // only held for a very short time, and gets grabbed at a very high frequency // (~100000 times per second). On Mac, the overhead of using a regular lock // is very high, see bug 1137963. static mozilla::Atomic<uintptr_t, mozilla::ReleaseAcquire> gTraceLogLocked; @@ -75,20 +75,29 @@ struct MOZ_STACK_CLASS AutoTraceLogLock while (!gTraceLogLocked.compareExchange(0, currentThread)) { PR_Sleep(PR_INTERVAL_NO_WAIT); /* yield */ } } } ~AutoTraceLogLock() { if (doRelease) gTraceLogLocked = 0; } }; -static PLHashTable* gBloatView; -static PLHashTable* gTypesToLog; -static PLHashTable* gObjectsToLog; -static PLHashTable* gSerialNumbers; +class BloatEntry; +struct SerialNumberRecord; + +using BloatHash = nsClassHashtable<nsDepCharHashKey, BloatEntry>; +using CharPtrSet = nsTHashtable<nsCharPtrHashKey>; +using IntPtrSet = nsTHashtable<IntPtrHashKey>; +using SerialHash = nsClassHashtable<nsVoidPtrHashKey, SerialNumberRecord>; + +static StaticAutoPtr<BloatHash> gBloatView; +static StaticAutoPtr<CharPtrSet> gTypesToLog; +static StaticAutoPtr<IntPtrSet> gObjectsToLog; +static StaticAutoPtr<SerialHash> gSerialNumbers; + static intptr_t gNextSerialNumber; static bool gDumpedStatistics = false; static bool gLogJSStacks = false; // By default, debug builds only do bloat logging. Bloat logging // only tries to record when an object is created or destroyed, so we // optimize the common case in NS_LogAddRef and NS_LogRelease where // only bloat logging is enabled and no logging needs to be done. @@ -200,66 +209,16 @@ AssertActivityIsLegal() # define ASSERT_ACTIVITY_IS_LEGAL \ do { \ AssertActivityIsLegal(); \ } while(0) #else # define ASSERT_ACTIVITY_IS_LEGAL do { } while(0) #endif // DEBUG -// These functions are copied from nsprpub/lib/ds/plhash.c, with changes -// to the functions not called Default* to free the SerialNumberRecord or -// the BloatEntry. - -static void* -DefaultAllocTable(void* aPool, size_t aSize) -{ - return malloc(aSize); -} - -static void -DefaultFreeTable(void* aPool, void* aItem) -{ - free(aItem); -} - -static PLHashEntry* -DefaultAllocEntry(void* aPool, const void* aKey) -{ - return (PLHashEntry*) malloc(sizeof(PLHashEntry)); -} - -static void -SerialNumberFreeEntry(void* aPool, PLHashEntry* aHashEntry, unsigned aFlag) -{ - if (aFlag == HT_FREE_ENTRY) { - delete static_cast<SerialNumberRecord*>(aHashEntry->value); - free(aHashEntry); - } -} - -static void -TypesToLogFreeEntry(void* aPool, PLHashEntry* aHashEntry, unsigned aFlag) -{ - if (aFlag == HT_FREE_ENTRY) { - free(const_cast<char*>(static_cast<const char*>(aHashEntry->key))); - free(aHashEntry); - } -} - -static const PLHashAllocOps serialNumberHashAllocOps = { - DefaultAllocTable, DefaultFreeTable, - DefaultAllocEntry, SerialNumberFreeEntry -}; - -static const PLHashAllocOps typesToLogHashAllocOps = { - DefaultAllocTable, DefaultFreeTable, - DefaultAllocEntry, TypesToLogFreeEntry -}; - //////////////////////////////////////////////////////////////////////////////// class CodeAddressServiceStringTable final { public: CodeAddressServiceStringTable() : mSet(32) {} const char* Intern(const char* aString) @@ -333,34 +292,16 @@ public: mStats.mCreates++; } void Dtor() { mStats.mDestroys++; } - static int DumpEntry(PLHashEntry* aHashEntry, int aIndex, void* aArg) - { - BloatEntry* entry = (BloatEntry*)aHashEntry->value; - if (entry) { - static_cast<nsTArray<BloatEntry*>*>(aArg)->AppendElement(entry); - } - return HT_ENUMERATE_NEXT; - } - - static int TotalEntries(PLHashEntry* aHashEntry, int aIndex, void* aArg) - { - BloatEntry* entry = (BloatEntry*)aHashEntry->value; - if (entry && nsCRT::strcmp(entry->mClassName, "TOTAL") != 0) { - entry->Total((BloatEntry*)aArg); - } - return HT_ENUMERATE_NEXT; - } - void Total(BloatEntry* aTotal) { aTotal->mStats.mCreates += mStats.mCreates; aTotal->mStats.mDestroys += mStats.mDestroys; aTotal->mClassSize += mClassSize * mStats.mCreates; // adjust for average in DumpTotal aTotal->mTotalLeaked += mClassSize * mStats.NumLeaked(); } @@ -407,88 +348,60 @@ public: protected: char* mClassName; double mClassSize; // This is stored as a double because of the way we compute the avg class size for total bloat. int64_t mTotalLeaked; // Used only for TOTAL entry. nsTraceRefcntStats mStats; }; static void -BloatViewFreeEntry(void* aPool, PLHashEntry* aHashEntry, unsigned aFlag) -{ - if (aFlag == HT_FREE_ENTRY) { - BloatEntry* entry = static_cast<BloatEntry*>(aHashEntry->value); - delete entry; - free(aHashEntry); - } -} - -const static PLHashAllocOps bloatViewHashAllocOps = { - DefaultAllocTable, DefaultFreeTable, - DefaultAllocEntry, BloatViewFreeEntry -}; - -static void RecreateBloatView() { - gBloatView = PL_NewHashTable(256, - PL_HashString, - PL_CompareStrings, - PL_CompareValues, - &bloatViewHashAllocOps, nullptr); + gBloatView = new BloatHash(256); } static BloatEntry* GetBloatEntry(const char* aTypeName, uint32_t aInstanceSize) { if (!gBloatView) { RecreateBloatView(); } - BloatEntry* entry = nullptr; - if (gBloatView) { - entry = (BloatEntry*)PL_HashTableLookup(gBloatView, aTypeName); - if (!entry && aInstanceSize > 0) { - - entry = new BloatEntry(aTypeName, aInstanceSize); - PLHashEntry* e = PL_HashTableAdd(gBloatView, aTypeName, entry); - if (!e) { - delete entry; - entry = nullptr; - } - } else { - MOZ_ASSERT(aInstanceSize == 0 || entry->GetClassSize() == aInstanceSize, - "Mismatched sizes were recorded in the memory leak logging table. " - "The usual cause of this is having a templated class that uses " - "MOZ_COUNT_{C,D}TOR in the constructor or destructor, respectively. " - "As a workaround, the MOZ_COUNT_{C,D}TOR calls can be moved to a " - "non-templated base class. Another possible cause is a runnable with " - "an mName that matches another refcounted class."); - } + BloatEntry* entry = gBloatView->Get(aTypeName); + if (!entry && aInstanceSize > 0) { + entry = new BloatEntry(aTypeName, aInstanceSize); + gBloatView->Put(aTypeName, entry); + } else { + MOZ_ASSERT(aInstanceSize == 0 || entry->GetClassSize() == aInstanceSize, + "Mismatched sizes were recorded in the memory leak logging table. " + "The usual cause of this is having a templated class that uses " + "MOZ_COUNT_{C,D}TOR in the constructor or destructor, respectively. " + "As a workaround, the MOZ_COUNT_{C,D}TOR calls can be moved to a " + "non-templated base class. Another possible cause is a runnable with " + "an mName that matches another refcounted class."); } return entry; } -static int -DumpSerialNumbers(PLHashEntry* aHashEntry, int aIndex, void* aClosure) +static void +DumpSerialNumbers(const SerialHash::Iterator& aHashEntry, FILE* aFd) { - SerialNumberRecord* record = - static_cast<SerialNumberRecord*>(aHashEntry->value); - auto* outputFile = static_cast<FILE*>(aClosure); + SerialNumberRecord* record = aHashEntry.Data(); + auto* outputFile = aFd; #ifdef HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR fprintf(outputFile, "%" PRIdPTR " @%p (%d references; %d from COMPtrs)\n", record->serialNumber, - aHashEntry->key, + aHashEntry.Key(), record->refCount, record->COMPtrCount); #else fprintf(outputFile, "%" PRIdPTR " @%p (%d references)\n", record->serialNumber, - aHashEntry->key, + aHashEntry.Key(), record->refCount); #endif if (!record->allocationStack.empty()) { static const size_t bufLen = 1024; char buf[bufLen]; fprintf(outputFile, "allocation stack:\n"); for (size_t i = 0, length = record->allocationStack.size(); i < length; @@ -501,18 +414,16 @@ DumpSerialNumbers(PLHashEntry* aHashEntr if (gLogJSStacks) { if (record->jsStack) { fprintf(outputFile, "JS allocation stack:\n%s\n", record->jsStack.get()); } else { fprintf(outputFile, "There is no JS context on the stack.\n"); } } - - return HT_ENUMERATE_NEXT; } template<> class nsDefaultComparator<BloatEntry*, BloatEntry*> { public: bool Equals(BloatEntry* const& aEntry1, BloatEntry* const& aEntry2) const @@ -540,27 +451,36 @@ nsTraceRefcnt::DumpStatistics() "bogus positive or negative leaks being reported"); gDumpedStatistics = true; // Don't try to log while we hold the lock, we'd deadlock. AutoRestore<LoggingType> saveLogging(gLogging); gLogging = NoLogging; BloatEntry total("TOTAL", 0); - PL_HashTableEnumerateEntries(gBloatView, BloatEntry::TotalEntries, &total); + for (auto iter = gBloatView->Iter(); !iter.Done(); iter.Next()) { + BloatEntry* entry = iter.Data(); + if (nsCRT::strcmp(entry->GetClassName(), "TOTAL") != 0) { + entry->Total(&total); + } + } + const char* msg; if (gLogLeaksOnly) { msg = "ALL (cumulative) LEAK STATISTICS"; } else { msg = "ALL (cumulative) LEAK AND BLOAT STATISTICS"; } const bool leaked = total.PrintDumpHeader(gBloatLog, msg); nsTArray<BloatEntry*> entries; - PL_HashTableEnumerateEntries(gBloatView, BloatEntry::DumpEntry, &entries); + for (auto iter = gBloatView->Iter(); !iter.Done(); iter.Next()) { + entries.AppendElement(iter.Data()); + } + const uint32_t count = entries.Length(); if (!gLogLeaksOnly || leaked) { // Sort the entries alphabetically by classname. entries.Sort(); for (uint32_t i = 0; i < count; ++i) { BloatEntry* entry = entries[i]; @@ -569,107 +489,62 @@ nsTraceRefcnt::DumpStatistics() fprintf(gBloatLog, "\n"); } fprintf(gBloatLog, "nsTraceRefcnt::DumpStatistics: %d entries\n", count); if (gSerialNumbers) { fprintf(gBloatLog, "\nSerial Numbers of Leaked Objects:\n"); - PL_HashTableEnumerateEntries(gSerialNumbers, DumpSerialNumbers, gBloatLog); + for (auto iter = gSerialNumbers->Iter(); !iter.Done(); iter.Next()) { + DumpSerialNumbers(iter, gBloatLog); + } } return NS_OK; } void nsTraceRefcnt::ResetStatistics() { AutoTraceLogLock lock; - if (gBloatView) { - PL_HashTableDestroy(gBloatView); - gBloatView = nullptr; - } -} - -static bool -LogThisType(const char* aTypeName) -{ - void* he = PL_HashTableLookup(gTypesToLog, aTypeName); - return he != nullptr; -} - -static PLHashNumber -HashNumber(const void* aKey) -{ - return PLHashNumber(NS_PTR_TO_INT32(aKey)); + gBloatView = nullptr; } static intptr_t GetSerialNumber(void* aPtr, bool aCreate) { - PLHashEntry** hep = PL_HashTableRawLookup(gSerialNumbers, - HashNumber(aPtr), - aPtr); - if (hep && *hep) { - MOZ_RELEASE_ASSERT(!aCreate, "If an object already has a serial number, we should be destroying it."); - return static_cast<SerialNumberRecord*>((*hep)->value)->serialNumber; + if (!aCreate) { + auto record = gSerialNumbers->Get(aPtr); + return record ? record->serialNumber : 0; } - if (!aCreate) { - return 0; + auto entry = gSerialNumbers->LookupForAdd(aPtr); + if (entry) { + MOZ_CRASH("If an object already has a serial number, we should be destroying it."); } - SerialNumberRecord* record = new SerialNumberRecord(); + auto record = entry.OrInsert([]() { return new SerialNumberRecord(); }); WalkTheStackSavingLocations(record->allocationStack); - PL_HashTableRawAdd(gSerialNumbers, hep, HashNumber(aPtr), - aPtr, static_cast<void*>(record)); if (gLogJSStacks) { record->SaveJSStack(); } return gNextSerialNumber; } -static int32_t* -GetRefCount(void* aPtr) -{ - PLHashEntry** hep = PL_HashTableRawLookup(gSerialNumbers, - HashNumber(aPtr), - aPtr); - if (hep && *hep) { - return &(static_cast<SerialNumberRecord*>((*hep)->value)->refCount); - } else { - return nullptr; - } -} - -#ifdef HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR -static int32_t* -GetCOMPtrCount(void* aPtr) -{ - PLHashEntry** hep = PL_HashTableRawLookup(gSerialNumbers, - HashNumber(aPtr), - aPtr); - if (hep && *hep) { - return &(static_cast<SerialNumberRecord*>((*hep)->value)->COMPtrCount); - } - return nullptr; -} -#endif // HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR - static void RecycleSerialNumberPtr(void* aPtr) { - PL_HashTableRemove(gSerialNumbers, aPtr); + gSerialNumbers->Remove(aPtr); } static bool LogThisObj(intptr_t aSerialNumber) { - return (bool)PL_HashTableLookup(gObjectsToLog, (const void*)aSerialNumber); + return gObjectsToLog->Contains(aSerialNumber); } using EnvCharType = mozilla::filesystem::Path::value_type; static bool InitLog(const EnvCharType* aEnvVar, const char* aMsg, FILE** aResult) { #ifdef XP_WIN @@ -797,64 +672,43 @@ InitTraceLog() } #endif // HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR #undef ENVVAR if (classes) { // if XPCOM_MEM_LOG_CLASSES was set to some value, the value is interpreted // as a list of class names to track - gTypesToLog = PL_NewHashTable(256, - PL_HashString, - PL_CompareStrings, - PL_CompareValues, - &typesToLogHashAllocOps, nullptr); - if (!gTypesToLog) { - NS_WARNING("out of memory"); - fprintf(stdout, "### XPCOM_MEM_LOG_CLASSES defined -- unable to log specific classes\n"); - } else { - fprintf(stdout, "### XPCOM_MEM_LOG_CLASSES defined -- only logging these classes: "); - const char* cp = classes; - for (;;) { - char* cm = (char*)strchr(cp, ','); - if (cm) { - *cm = '\0'; - } - PL_HashTableAdd(gTypesToLog, strdup(cp), (void*)1); - fprintf(stdout, "%s ", cp); - if (!cm) { - break; - } - *cm = ','; - cp = cm + 1; + gTypesToLog = new CharPtrSet(256); + + fprintf(stdout, "### XPCOM_MEM_LOG_CLASSES defined -- only logging these classes: "); + const char* cp = classes; + for (;;) { + char* cm = (char*)strchr(cp, ','); + if (cm) { + *cm = '\0'; } - fprintf(stdout, "\n"); + gTypesToLog->PutEntry(cp); + fprintf(stdout, "%s ", cp); + if (!cm) { + break; + } + *cm = ','; + cp = cm + 1; } + fprintf(stdout, "\n"); - gSerialNumbers = PL_NewHashTable(256, - HashNumber, - PL_CompareValues, - PL_CompareValues, - &serialNumberHashAllocOps, nullptr); - - + gSerialNumbers = new SerialHash(256); } const char* objects = getenv("XPCOM_MEM_LOG_OBJECTS"); if (objects) { - gObjectsToLog = PL_NewHashTable(256, - HashNumber, - PL_CompareValues, - PL_CompareValues, - nullptr, nullptr); + gObjectsToLog = new IntPtrSet(256); - if (!gObjectsToLog) { - NS_WARNING("out of memory"); - fprintf(stdout, "### XPCOM_MEM_LOG_OBJECTS defined -- unable to log specific objects\n"); - } else if (!(gRefcntsLog || gAllocLog || gCOMPtrLog)) { + if (!(gRefcntsLog || gAllocLog || gCOMPtrLog)) { fprintf(stdout, "### XPCOM_MEM_LOG_OBJECTS defined -- but none of XPCOM_MEM_(REFCNT|ALLOC|COMPTR)_LOG is defined\n"); } else { fprintf(stdout, "### XPCOM_MEM_LOG_OBJECTS defined -- only logging these objects: "); const char* cp = objects; for (;;) { char* cm = (char*)strchr(cp, ','); if (cm) { *cm = '\0'; @@ -870,17 +724,17 @@ InitTraceLog() top *= 10; top += *cp - '0'; ++cp; } if (!bottom) { bottom = top; } for (intptr_t serialno = bottom; serialno <= top; serialno++) { - PL_HashTableAdd(gObjectsToLog, (const void*)serialno, (void*)1); + gObjectsToLog->PutEntry(serialno); fprintf(stdout, "%" PRIdPTR " ", serialno); } if (!cm) { break; } *cm = ','; cp = cm + 1; } @@ -1088,28 +942,27 @@ NS_LogAddRef(void* aPtr, nsrefcnt aRefcn if (entry) { entry->Ctor(); } } // Here's the case where MOZ_COUNT_CTOR was not used, // yet we still want to see creation information: - bool loggingThisType = (!gTypesToLog || LogThisType(aClass)); + bool loggingThisType = (!gTypesToLog || gTypesToLog->Contains(aClass)); intptr_t serialno = 0; if (gSerialNumbers && loggingThisType) { serialno = GetSerialNumber(aPtr, aRefcnt == 1); MOZ_ASSERT(serialno != 0, "Serial number requested for unrecognized pointer! " "Are you memmoving a refcounted object?"); - int32_t* count = GetRefCount(aPtr); - if (count) { - (*count)++; + auto record = gSerialNumbers->Get(aPtr); + if (record) { + ++record->refCount; } - } bool loggingThisObject = (!gObjectsToLog || LogThisObj(serialno)); if (aRefcnt == 1 && gAllocLog && loggingThisType && loggingThisObject) { fprintf(gAllocLog, "\n<%s> %p %" PRIdPTR " Create [thread %p]\n", aClass, aPtr, serialno, PR_GetCurrentThread()); WalkTheStackCached(gAllocLog); } @@ -1138,28 +991,27 @@ NS_LogRelease(void* aPtr, nsrefcnt aRefc if (aRefcnt == 0 && gBloatLog) { BloatEntry* entry = GetBloatEntry(aClass, 0); if (entry) { entry->Dtor(); } } - bool loggingThisType = (!gTypesToLog || LogThisType(aClass)); + bool loggingThisType = (!gTypesToLog || gTypesToLog->Contains(aClass)); intptr_t serialno = 0; if (gSerialNumbers && loggingThisType) { serialno = GetSerialNumber(aPtr, false); MOZ_ASSERT(serialno != 0, "Serial number requested for unrecognized pointer! " "Are you memmoving a refcounted object?"); - int32_t* count = GetRefCount(aPtr); - if (count) { - (*count)--; + auto record = gSerialNumbers->Get(aPtr); + if (record) { + --record->refCount; } - } bool loggingThisObject = (!gObjectsToLog || LogThisObj(serialno)); if (gRefcntsLog && loggingThisType && loggingThisObject) { // Can't use MOZ_LOG(), b/c it truncates the line fprintf(gRefcntsLog, "\n<%s> %p %" PRIuPTR " Release %" PRIuPTR " [thread %p]\n", aClass, aPtr, serialno, aRefcnt, PR_GetCurrentThread()); @@ -1197,17 +1049,17 @@ NS_LogCtor(void* aPtr, const char* aType if (gBloatLog) { BloatEntry* entry = GetBloatEntry(aType, aInstanceSize); if (entry) { entry->Ctor(); } } - bool loggingThisType = (!gTypesToLog || LogThisType(aType)); + bool loggingThisType = (!gTypesToLog || gTypesToLog->Contains(aType)); intptr_t serialno = 0; if (gSerialNumbers && loggingThisType) { serialno = GetSerialNumber(aPtr, true); MOZ_ASSERT(serialno != 0, "GetSerialNumber should never return 0 when passed true"); } bool loggingThisObject = (!gObjectsToLog || LogThisObj(serialno)); if (gAllocLog && loggingThisType && loggingThisObject) { @@ -1234,17 +1086,17 @@ NS_LogDtor(void* aPtr, const char* aType if (gBloatLog) { BloatEntry* entry = GetBloatEntry(aType, aInstanceSize); if (entry) { entry->Dtor(); } } - bool loggingThisType = (!gTypesToLog || LogThisType(aType)); + bool loggingThisType = (!gTypesToLog || gTypesToLog->Contains(aType)); intptr_t serialno = 0; if (gSerialNumbers && loggingThisType) { serialno = GetSerialNumber(aPtr, false); MOZ_ASSERT(serialno != 0, "Serial number requested for unrecognized pointer! " "Are you memmoving a MOZ_COUNT_CTOR-tracked object?"); RecycleSerialNumberPtr(aPtr); } @@ -1280,26 +1132,23 @@ NS_LogCOMPtrAddRef(void* aCOMPtr, nsISup if (gLogging == FullLogging) { AutoTraceLogLock lock; intptr_t serialno = GetSerialNumber(object, false); if (serialno == 0) { return; } - int32_t* count = GetCOMPtrCount(object); - if (count) { - (*count)++; - } - + auto record = gSerialNumbers->Get(object); + int32_t count = record ? ++record->COMPtrCount : -1; bool loggingThisObject = (!gObjectsToLog || LogThisObj(serialno)); if (gCOMPtrLog && loggingThisObject) { fprintf(gCOMPtrLog, "\n<?> %p %" PRIdPTR " nsCOMPtrAddRef %d %p\n", - object, serialno, count ? (*count) : -1, aCOMPtr); + object, serialno, count, aCOMPtr); WalkTheStackCached(gCOMPtrLog); } } #endif // HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR } EXPORT_XPCOM_API(void) @@ -1321,52 +1170,37 @@ NS_LogCOMPtrRelease(void* aCOMPtr, nsISu if (gLogging == FullLogging) { AutoTraceLogLock lock; intptr_t serialno = GetSerialNumber(object, false); if (serialno == 0) { return; } - int32_t* count = GetCOMPtrCount(object); - if (count) { - (*count)--; - } - + auto record = gSerialNumbers->Get(object); + int32_t count = record ? --record->COMPtrCount : -1; bool loggingThisObject = (!gObjectsToLog || LogThisObj(serialno)); if (gCOMPtrLog && loggingThisObject) { fprintf(gCOMPtrLog, "\n<?> %p %" PRIdPTR " nsCOMPtrRelease %d %p\n", - object, serialno, count ? (*count) : -1, aCOMPtr); + object, serialno, count, aCOMPtr); WalkTheStackCached(gCOMPtrLog); } } #endif // HAVE_CPP_DYNAMIC_CAST_TO_VOID_PTR } void nsTraceRefcnt::Shutdown() { gCodeAddressService = nullptr; - if (gBloatView) { - PL_HashTableDestroy(gBloatView); - gBloatView = nullptr; - } - if (gTypesToLog) { - PL_HashTableDestroy(gTypesToLog); - gTypesToLog = nullptr; - } - if (gObjectsToLog) { - PL_HashTableDestroy(gObjectsToLog); - gObjectsToLog = nullptr; - } - if (gSerialNumbers) { - PL_HashTableDestroy(gSerialNumbers); - gSerialNumbers = nullptr; - } + gBloatView = nullptr; + gTypesToLog = nullptr; + gObjectsToLog = nullptr; + gSerialNumbers = nullptr; maybeUnregisterAndCloseFile(gBloatLog); maybeUnregisterAndCloseFile(gRefcntsLog); maybeUnregisterAndCloseFile(gAllocLog); maybeUnregisterAndCloseFile(gCOMPtrLog); } void nsTraceRefcnt::SetActivityIsLegal(bool aLegal)
--- a/xpcom/ds/nsHashKeys.h +++ b/xpcom/ds/nsHashKeys.h @@ -15,16 +15,17 @@ #include "PLDHashTable.h" #include <new> #include "nsString.h" #include "nsCRTGlue.h" #include "nsUnicharUtils.h" #include "nsPointerHashKeys.h" +#include <stdint.h> #include <stdlib.h> #include <string.h> #include "mozilla/HashFunctions.h" #include "mozilla/Move.h" namespace mozilla { @@ -49,16 +50,17 @@ HashString(const nsACString& aStr) * classes follows the nsTHashtable::EntryType specification * * Lightweight keytypes provided here: * nsStringHashKey * nsCStringHashKey * nsUint32HashKey * nsUint64HashKey * nsFloatHashKey + * IntPtrHashKey * nsPtrHashKey * nsClearingPtrHashKey * nsVoidPtrHashKey * nsClearingVoidPtrHashKey * nsISupportsHashKey * nsIDHashKey * nsDepCharHashKey * nsCharPtrHashKey @@ -280,16 +282,45 @@ public: } enum { ALLOW_MEMMOVE = true }; private: const float mValue; }; /** + * hashkey wrapper using intptr_t KeyType + * + * @see nsTHashtable::EntryType for specification + */ +class IntPtrHashKey : public PLDHashEntryHdr +{ +public: + typedef const intptr_t& KeyType; + typedef const intptr_t* KeyTypePointer; + + explicit IntPtrHashKey(KeyTypePointer aKey) : mValue(*aKey) {} + IntPtrHashKey(const IntPtrHashKey& aToCopy) : mValue(aToCopy.mValue) {} + ~IntPtrHashKey() {} + + KeyType GetKey() const { return mValue; } + bool KeyEquals(KeyTypePointer aKey) const { return *aKey == mValue; } + + static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } + static PLDHashNumber HashKey(KeyTypePointer aKey) + { + return mozilla::HashGeneric(*aKey); + } + enum { ALLOW_MEMMOVE = true }; + +private: + const intptr_t mValue; +}; + +/** * hashkey wrapper using nsISupports* KeyType * * @see nsTHashtable::EntryType for specification */ class nsISupportsHashKey : public PLDHashEntryHdr { public: typedef nsISupports* KeyType;
--- a/xpcom/threads/nsIThreadManager.idl +++ b/xpcom/threads/nsIThreadManager.idl @@ -24,65 +24,65 @@ interface nsINestedEventLoopCondition : /** * An interface for creating and locating nsIThread instances. */ [scriptable, uuid(1be89eca-e2f7-453b-8d38-c11ba247f6f3)] interface nsIThreadManager : nsISupports { /** * Default number of bytes reserved for a thread's stack, if no stack size - * is specified in newThread(). 0 means use platform default. - */ - const unsigned long DEFAULT_STACK_SIZE = 0; - -%{C++ - /* DEFAULT_STACK_SIZE can be a little overzealous for many platforms. On - * Linux and OS X, for instance, the default thread stack size is whatever - * getrlimit(RLIMIT_STACK) returns, which is often set at 8MB. The - * default on Windows is 1MB, which is a little more reasonable. But - * for thread pools, each individual thread often doesn't need that much - * stack space. + * is specified in newThread(). + * + * Defaults can be a little overzealous for many platforms. + * + * On Linux and OS X, for instance, the default thread stack size is whatever + * getrlimit(RLIMIT_STACK) returns, which is often set at 8MB. Or, on Linux, + * if the stack size is unlimited, we fall back to 2MB. This causes particular + * problems on Linux, which allocates 2MB huge VM pages, and will often + * immediately allocate them for any stacks which are 2MB or larger. * - * We therefore have a separate setting for a reasonable stack size for - * a thread pool worker thread. + * The default on Windows is 1MB, which is a little more reasonable. But the + * vast majority of our threads don't need anywhere near that much space. + * + * ASan and TSan builds, however, often need a bit more, so give them a the + * platform default. */ +%{C++ #if defined(MOZ_ASAN) || defined(MOZ_TSAN) - // Use the system default in ASAN builds, because the default is assumed - // to be larger than the size we want to use and is hopefully sufficient - // for ASAN. + static constexpr uint32_t DEFAULT_STACK_SIZE = 0; +#else + static constexpr uint32_t DEFAULT_STACK_SIZE = 256 * 1024; +#endif + static const uint32_t kThreadPoolStackSize = DEFAULT_STACK_SIZE; -#elif defined(XP_WIN) || defined(XP_MACOSX) || defined(LINUX) - static const uint32_t kThreadPoolStackSize = (256 * 1024); -#else - // All other platforms use their system default. - static const uint32_t kThreadPoolStackSize = DEFAULT_STACK_SIZE; -#endif %} /** * Create a new thread (a global, user PRThread). * * @param creationFlags * Reserved for future use. Pass 0. * @param stackSize - * Number of bytes to reserve for the thread's stack. + * Number of bytes to reserve for the thread's stack. 0 means use platform + * default. * * @returns * The newly created nsIThread object. */ nsIThread newThread(in unsigned long creationFlags, [optional] in unsigned long stackSize); /** * Create a new thread (a global, user PRThread) with the specified name. * * @param name * The name of the thread. Passing an empty name is equivalent to * calling newThread(0, stackSize), i.e. the thread will not be named. * @param stackSize - * Number of bytes to reserve for the thread's stack. + * Number of bytes to reserve for the thread's stack. 0 means use platform + * default. * * @returns * The newly created nsIThread object. */ [noscript] nsIThread newNamedThread(in ACString name, [optional] in unsigned long stackSize); /** * Get the nsIThread object (if any) corresponding to the given PRThread.