Bug 1426366 - Detect http transaction stalls with TFO. r=hurley
authorDragana Damjanovic <dd.mozilla@gmail.com>
Thu, 21 Dec 2017 18:32:28 +0100
changeset 397224 e288b04ea679b4d2fbdc52e1b4ae678c39f3ea91
parent 397223 c1a62b8156074427354dab1aea2780b41c058247
child 397225 bfda0655abefe96268ef1cc61cc246b1351af124
push id33128
push useraiakab@mozilla.com
push dateThu, 21 Dec 2017 22:20:14 +0000
treeherdermozilla-central@f844e99f26b1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershurley
bugs1426366
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1426366 - Detect http transaction stalls with TFO. r=hurley
modules/libpref/init/all.js
netwerk/protocol/http/Http2Session.cpp
netwerk/protocol/http/Http2Session.h
netwerk/protocol/http/nsHttpConnection.cpp
netwerk/protocol/http/nsHttpConnection.h
netwerk/protocol/http/nsHttpHandler.cpp
netwerk/protocol/http/nsHttpHandler.h
toolkit/components/telemetry/Histograms.json
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -4779,16 +4779,26 @@ pref("network.tcp.keepalive.retry_interv
 // Default maximum probe retransmissions.
 // Linux only; not configurable on Win and Mac; fixed at 10 and 8 respectively.
 #if defined(XP_UNIX) && !defined(XP_MACOSX)
 pref("network.tcp.keepalive.probe_count", 4);
 #endif
 
 pref("network.tcp.tcp_fastopen_enable", false);
 pref("network.tcp.tcp_fastopen_consecutive_failure_limit", 5);
+// We are trying to detect stalled tcp connections that use TFO and TLS
+// (bug 1395494).
+// This is only happening if a connection is idle for more than 10s, but we
+// will make this a pref. If tcp_fastopen_http_stalls_limit of stalls are
+// detected the TCP fast open will be disabled.
+// If tcp_fastopen_http_check_for_stalls_only_if_idle_for is set to 0 the
+// check will not be performed.
+pref("network.tcp.tcp_fastopen_http_check_for_stalls_only_if_idle_for", 10);
+pref("network.tcp.tcp_fastopen_http_stalls_limit", 3);
+pref("network.tcp.tcp_fastopen_http_stalls_timeout", 20);
 
 // Whether to disable acceleration for all widgets.
 pref("layers.acceleration.disabled", false);
 // Preference that when switched at runtime will run a series of benchmarks
 // and output the result to stderr.
 pref("layers.bench.enabled", false);
 
 #if defined(XP_WIN)
--- a/netwerk/protocol/http/Http2Session.cpp
+++ b/netwerk/protocol/http/Http2Session.cpp
@@ -37,16 +37,17 @@
 #include "mozilla/Sprintf.h"
 #include "nsSocketTransportService2.h"
 #include "nsNetUtil.h"
 #include "nsICacheEntry.h"
 #include "nsICacheStorageService.h"
 #include "nsICacheStorage.h"
 #include "CacheControlParser.h"
 #include "LoadContextInfo.h"
+#include "TCPFastOpenLayer.h"
 
 namespace mozilla {
 namespace net {
 
 // Http2Session has multiple inheritance of things that implement nsISupports
 NS_IMPL_ADDREF(Http2Session)
 NS_IMPL_RELEASE(Http2Session)
 NS_INTERFACE_MAP_BEGIN(Http2Session)
@@ -114,16 +115,18 @@ Http2Session::Http2Session(nsISocketTran
   , mPingSentEpoch(0)
   , mPreviousUsed(false)
   , mWaitingForSettingsAck(false)
   , mGoAwayOnPush(false)
   , mUseH2Deps(false)
   , mAttemptingEarlyData(attemptingEarlyData)
   , mOriginFrameActivated(false)
   , mTlsHandshakeFinished(false)
+  , mCheckNetworkStallsWithTFO(false)
+  , mLastRequestBytesSentTime(0)
 {
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   static uint64_t sSerial;
   mSerial = ++sSerial;
 
   LOG3(("Http2Session::Http2Session %p serial=0x%" PRIX64 "\n", this, mSerial));
 
@@ -264,32 +267,43 @@ Http2Session::IdleTime()
 uint32_t
 Http2Session::ReadTimeoutTick(PRIntervalTime now)
 {
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   LOG3(("Http2Session::ReadTimeoutTick %p delta since last read %ds\n",
        this, PR_IntervalToSeconds(now - mLastReadEpoch)));
 
+  uint32_t nextTick = UINT32_MAX;
+  if (mCheckNetworkStallsWithTFO && mLastRequestBytesSentTime) {
+    PRIntervalTime initialResponseDelta = now - mLastRequestBytesSentTime;
+    if (initialResponseDelta >= gHttpHandler->FastOpenStallsTimeout()) {
+      gHttpHandler->IncrementFastOpenStallsCounter();
+      mCheckNetworkStallsWithTFO = false;
+    } else {
+      nextTick = PR_IntervalToSeconds(gHttpHandler->FastOpenStallsTimeout()) -
+                 PR_IntervalToSeconds(initialResponseDelta);
+    }
+  }
   if (!mPingThreshold)
-    return UINT32_MAX;
+    return nextTick;
 
   if ((now - mLastReadEpoch) < mPingThreshold) {
     // recent activity means ping is not an issue
     if (mPingSentEpoch) {
       mPingSentEpoch = 0;
       if (mPreviousUsed) {
         // restore the former value
         mPingThreshold = mPreviousPingThreshold;
         mPreviousUsed = false;
       }
     }
 
-    return PR_IntervalToSeconds(mPingThreshold) -
-      PR_IntervalToSeconds(now - mLastReadEpoch);
+    return std::min(nextTick, PR_IntervalToSeconds(mPingThreshold) -
+                              PR_IntervalToSeconds(now - mLastReadEpoch));
   }
 
   if (mPingSentEpoch) {
     LOG3(("Http2Session::ReadTimeoutTick %p handle outstanding ping\n", this));
     if ((now - mPingSentEpoch) >= gHttpHandler->SpdyPingTimeout()) {
       LOG3(("Http2Session::ReadTimeoutTick %p Ping Timer Exhaustion\n", this));
       mPingSentEpoch = 0;
       Close(NS_ERROR_NET_TIMEOUT);
@@ -368,16 +382,30 @@ Http2Session::RegisterStreamID(Http2Stre
   if (mStreamIDHash.Get(aNewID)) {
     LOG3(("   New ID already present\n"));
     MOZ_ASSERT(false, "New ID already present in mStreamIDHash");
     mShouldGoAway = true;
     return kDeadStreamID;
   }
 
   mStreamIDHash.Put(aNewID, stream);
+
+  // If TCP fast Open has been used and conection was idle for some time
+  // we will be cautious and watch out for bug 1395494.
+  if (!mCheckNetworkStallsWithTFO && mConnection) {
+    RefPtr<nsHttpConnection> conn = mConnection->HttpConnection();
+    if (conn && (conn->GetFastOpenStatus() == TFO_DATA_SENT) &&
+        gHttpHandler->CheckIfConnectionIsStalledOnlyIfIdleForThisAmountOfSeconds() &&
+        IdleTime() >= gHttpHandler->CheckIfConnectionIsStalledOnlyIfIdleForThisAmountOfSeconds()) {
+      // If a connection was using the TCP FastOpen and it was idle for a
+      // long time we should check for stalls like bug 1395494.
+      mCheckNetworkStallsWithTFO = true;
+      mLastRequestBytesSentTime = PR_IntervalNow();
+    }
+  }
   return aNewID;
 }
 
 bool
 Http2Session::AddStream(nsAHttpTransaction *aHttpTransaction,
                         int32_t aPriority,
                         bool aUseTunnel,
                         nsIInterfaceRequestor *aCallbacks)
@@ -507,18 +535,20 @@ Http2Session::NetworkRead(nsAHttpSegment
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   if (!count) {
     *countWritten = 0;
     return NS_OK;
   }
 
   nsresult rv = writer->OnWriteSegment(buf, count, countWritten);
-  if (NS_SUCCEEDED(rv) && *countWritten > 0)
+  if (NS_SUCCEEDED(rv) && *countWritten > 0) {
     mLastReadEpoch = PR_IntervalNow();
+    mCheckNetworkStallsWithTFO = false;
+  }
   return rv;
 }
 
 void
 Http2Session::SetWriteCallbacks()
 {
   if (mConnection &&
       (GetWriteQueueSize() || (mOutputQueueUsed > mOutputQueueSent))) {
--- a/netwerk/protocol/http/Http2Session.h
+++ b/netwerk/protocol/http/Http2Session.h
@@ -558,16 +558,19 @@ private:
     nsHttpRequestHead mRequestHead;
   };
 
   // A h2 session will be created before all socket events are trigered,
   // e.g. NS_NET_STATUS_TLS_HANDSHAKE_ENDED and for TFO many others.
   // We should propagate this events to the first nsHttpTransaction.
   RefPtr<nsHttpTransaction> mFirstHttpTransaction;
   bool mTlsHandshakeFinished;
+
+  bool mCheckNetworkStallsWithTFO;
+  PRIntervalTime mLastRequestBytesSentTime;
 private:
 /// connect tunnels
   void DispatchOnTunnel(nsAHttpTransaction *, nsIInterfaceRequestor *);
   void CreateTunnel(nsHttpTransaction *, nsHttpConnectionInfo *, nsIInterfaceRequestor *);
   void RegisterTunnel(Http2Stream *);
   void UnRegisterTunnel(Http2Stream *);
   uint32_t FindTunnelCount(nsHttpConnectionInfo *);
   nsDataHashtable<nsCStringHashKey, uint32_t> mTunnelHash;
--- a/netwerk/protocol/http/nsHttpConnection.cpp
+++ b/netwerk/protocol/http/nsHttpConnection.cpp
@@ -85,16 +85,18 @@ nsHttpConnection::nsHttpConnection()
     , mWaitingFor0RTTResponse(false)
     , mContentBytesWritten0RTT(0)
     , mEarlyDataNegotiated(false)
     , mDid0RTTSpdy(false)
     , mFastOpen(false)
     , mFastOpenStatus(TFO_NOT_SET)
     , mForceSendDuringFastOpenPending(false)
     , mReceivedSocketWouldBlockDuringFastOpen(false)
+    , mCheckNetworkStallsWithTFO(false)
+    , mLastRequestBytesSentTime(0)
 {
     LOG(("Creating nsHttpConnection @%p\n", this));
 
     // the default timeout is for when this connection has not yet processed a
     // transaction
     static const PRIntervalTime k5Sec = PR_SecondsToInterval(5);
     mIdleTimeout =
         (k5Sec < gHttpHandler->IdleTimeout()) ? k5Sec : gHttpHandler->IdleTimeout();
@@ -636,16 +638,28 @@ nsHttpConnection::Activate(nsAHttpTransa
     mPriority = pri;
     if (mTransaction && mUsingSpdyVersion) {
         return AddTransaction(trans, pri);
     }
 
     NS_ENSURE_ARG_POINTER(trans);
     NS_ENSURE_TRUE(!mTransaction, NS_ERROR_IN_PROGRESS);
 
+    // If TCP fast Open has been used and conection was idle for some time
+    // we will be cautious and watch out for bug 1395494.
+    if (mNPNComplete && (mFastOpenStatus == TFO_DATA_SENT) &&
+        gHttpHandler->CheckIfConnectionIsStalledOnlyIfIdleForThisAmountOfSeconds() &&
+        IdleTime() >= gHttpHandler->CheckIfConnectionIsStalledOnlyIfIdleForThisAmountOfSeconds()) {
+        // If a connection was using the TCP FastOpen and it was idle for a
+        // long time we should check for stalls like bug 1395494.
+        mCheckNetworkStallsWithTFO = true;
+        // Also reset last write. We should start measuring a stall time only
+        // after we really write a request to the network.
+        mLastRequestBytesSentTime = 0;
+    }
     // reset the read timers to wash away any idle time
     mLastWriteTime = mLastReadTime = PR_IntervalNow();
 
     // Connection failures are Activated() just like regular transacions.
     // If we don't have a confirmation of a connected socket then test it
     // with a write() to get relevant error code.
     if (!mConnectedTransport) {
         uint32_t count;
@@ -1350,16 +1364,29 @@ nsHttpConnection::ReadTimeoutTick(PRInte
             CloseTransaction(mTransaction, NS_ERROR_NET_TIMEOUT);
             return UINT32_MAX;
         }
         nextTickAfter = PR_IntervalToSeconds(mTransaction->ResponseTimeout()) -
                         PR_IntervalToSeconds(initialResponseDelta);
         nextTickAfter = std::max(nextTickAfter, 1U);
     }
 
+    // Check for the TCP Fast Open related stalls.
+    if (mCheckNetworkStallsWithTFO && mLastRequestBytesSentTime) {
+        PRIntervalTime initialResponseDelta = now - mLastRequestBytesSentTime;
+        if (initialResponseDelta >= gHttpHandler->FastOpenStallsTimeout()) {
+            gHttpHandler->IncrementFastOpenStallsCounter();
+            mCheckNetworkStallsWithTFO = false;
+        } else {
+            uint32_t next = PR_IntervalToSeconds(gHttpHandler->FastOpenStallsTimeout()) -
+                            PR_IntervalToSeconds(initialResponseDelta);
+            nextTickAfter = std::min(nextTickAfter, next);
+        }
+    }
+
     if (!mNPNComplete) {
       // We can reuse mLastWriteTime here, because it is set when the
       // connection is activated and only change when a transaction
       // succesfullu write to the socket and this can only happen after
       // the TLS handshake is done.
       PRIntervalTime initialTLSDelta = now - mLastWriteTime;
       if (initialTLSDelta > gHttpHandler->TLSHandshakeTimeout()) {
         LOG(("canceling transaction: tls handshake takes too long: tls handshake "
@@ -1851,16 +1878,19 @@ nsHttpConnection::OnSocketWritable()
                 // at this point we've written out the entire transaction, and now we
                 // must wait for the server's response.  we manufacture a status message
                 // here to reflect the fact that we are waiting.  this message will be
                 // trumped (overwritten) if the server responds quickly.
                 //
                 mTransaction->OnTransportStatus(mSocketTransport,
                                                 NS_NET_STATUS_WAITING_FOR,
                                                 0);
+                if (mCheckNetworkStallsWithTFO) {
+                    mLastRequestBytesSentTime = PR_IntervalNow();
+                }
 
                 rv = ResumeRecv(); // start reading
             }
             again = false;
         } else if (writeAttempts >= maxWriteAttempts) {
             LOG(("  yield for other transactions\n"));
             rv = mSocketOut->AsyncWait(this, 0, 0, nullptr); // continue writing
             again = false;
@@ -1893,16 +1923,18 @@ nsHttpConnection::OnWriteSegment(char *b
     nsresult rv = mSocketIn->Read(buf, count, countWritten);
     if (NS_FAILED(rv))
         mSocketInCondition = rv;
     else if (*countWritten == 0)
         mSocketInCondition = NS_BASE_STREAM_CLOSED;
     else
         mSocketInCondition = NS_OK; // reset condition
 
+    mCheckNetworkStallsWithTFO = false;
+
     return mSocketInCondition;
 }
 
 nsresult
 nsHttpConnection::OnSocketReadable()
 {
     LOG(("nsHttpConnection::OnSocketReadable [this=%p]\n", this));
 
--- a/netwerk/protocol/http/nsHttpConnection.h
+++ b/netwerk/protocol/http/nsHttpConnection.h
@@ -229,16 +229,19 @@ public:
     }
     // override of nsAHttpConnection
     virtual uint32_t Version();
 
     bool TestJoinConnection(const nsACString &hostname, int32_t port);
     bool JoinConnection(const nsACString &hostname, int32_t port);
 
     void SetFastOpenStatus(uint8_t tfoStatus);
+    uint8_t GetFastOpenStatus() {
+      return mFastOpenStatus;
+    }
 
     void SetEvent(nsresult aStatus);
 
 private:
     // Value (set in mTCPKeepaliveConfig) indicates which set of prefs to use.
     enum TCPKeepaliveConfig {
       kTCPKeepaliveDisabled = 0,
       kTCPKeepaliveShortLivedConfig,
@@ -396,16 +399,18 @@ private:
     nsCString                      mEarlyNegotiatedALPN;
     bool                           mDid0RTTSpdy;
 
     bool                           mFastOpen;
     uint8_t                        mFastOpenStatus;
 
     bool                           mForceSendDuringFastOpenPending;
     bool                           mReceivedSocketWouldBlockDuringFastOpen;
+    bool                           mCheckNetworkStallsWithTFO;
+    PRIntervalTime                 mLastRequestBytesSentTime;
 
 public:
     void BootstrapTimings(TimingStruct times);
 private:
     TimingStruct    mBootstrappedTimings;
 };
 
 NS_DEFINE_STATIC_IID_ACCESSOR(nsHttpConnection, NS_HTTPCONNECTION_IID)
--- a/netwerk/protocol/http/nsHttpHandler.cpp
+++ b/netwerk/protocol/http/nsHttpHandler.cpp
@@ -104,18 +104,21 @@
 #define BROWSER_PREF_PREFIX     "browser.cache."
 #define DONOTTRACK_HEADER_ENABLED "privacy.donottrackheader.enabled"
 #define H2MANDATORY_SUITE        "security.ssl3.ecdhe_rsa_aes_128_gcm_sha256"
 #define TELEMETRY_ENABLED        "toolkit.telemetry.enabled"
 #define ALLOW_EXPERIMENTS        "network.allow-experiments"
 #define SAFE_HINT_HEADER_VALUE   "safeHint.enabled"
 #define SECURITY_PREFIX          "security."
 
-#define TCP_FAST_OPEN_ENABLE        "network.tcp.tcp_fastopen_enable"
-#define TCP_FAST_OPEN_FAILURE_LIMIT "network.tcp.tcp_fastopen_consecutive_failure_limit"
+#define TCP_FAST_OPEN_ENABLE         "network.tcp.tcp_fastopen_enable"
+#define TCP_FAST_OPEN_FAILURE_LIMIT  "network.tcp.tcp_fastopen_consecutive_failure_limit"
+#define TCP_FAST_OPEN_STALLS_LIMIT   "network.tcp.tcp_fastopen_http_stalls_limit"
+#define TCP_FAST_OPEN_STALLS_IDLE    "network.tcp.tcp_fastopen_http_check_for_stalls_only_if_idle_for"
+#define TCP_FAST_OPEN_STALLS_TIMEOUT "network.tcp.tcp_fastopen_http_stalls_timeout"
 
 #define UA_PREF(_pref) UA_PREF_PREFIX _pref
 #define HTTP_PREF(_pref) HTTP_PREF_PREFIX _pref
 #define BROWSER_PREF(_pref) BROWSER_PREF_PREFIX _pref
 
 #define NS_HTTP_PROTOCOL_FLAGS (URI_STD | ALLOWS_PROXY | ALLOWS_PROXY_HTTP | URI_LOADABLE_BY_ANYONE)
 
 //-----------------------------------------------------------------------------
@@ -280,16 +283,20 @@ nsHttpHandler::nsHttpHandler()
     , mTCPKeepaliveLongLivedIdleTimeS(600)
     , mEnforceH1Framing(FRAMECHECK_BARELY)
     , mDefaultHpackBuffer(4096)
     , mMaxHttpResponseHeaderSize(393216)
     , mFocusedWindowTransactionRatio(0.9f)
     , mUseFastOpen(true)
     , mFastOpenConsecutiveFailureLimit(5)
     , mFastOpenConsecutiveFailureCounter(0)
+    , mFastOpenStallsLimit(3)
+    , mFastOpenStallsCounter(0)
+    , mFastOpenStallsIdleTime(10)
+    , mFastOpenStallsTimeout(20)
     , mActiveTabPriority(true)
     , mProcessId(0)
     , mNextChannelId(1)
     , mLastActiveTabLoadOptimizationLock("nsHttpConnectionMgr::LastActiveTabLoadOptimization")
 {
     LOG(("Creating nsHttpHandler [this=%p].\n", this));
 
     mUserAgentOverride.SetIsVoid(true);
@@ -447,16 +454,19 @@ nsHttpHandler::Init()
         prefBranch->AddObserver(TELEMETRY_ENABLED, this, true);
         prefBranch->AddObserver(H2MANDATORY_SUITE, this, true);
         prefBranch->AddObserver(HTTP_PREF("tcp_keepalive.short_lived_connections"), this, true);
         prefBranch->AddObserver(HTTP_PREF("tcp_keepalive.long_lived_connections"), this, true);
         prefBranch->AddObserver(SAFE_HINT_HEADER_VALUE, this, true);
         prefBranch->AddObserver(SECURITY_PREFIX, this, true);
         prefBranch->AddObserver(TCP_FAST_OPEN_ENABLE, this, true);
         prefBranch->AddObserver(TCP_FAST_OPEN_FAILURE_LIMIT, this, true);
+        prefBranch->AddObserver(TCP_FAST_OPEN_STALLS_LIMIT, this, true);
+        prefBranch->AddObserver(TCP_FAST_OPEN_STALLS_IDLE, this, true);
+        prefBranch->AddObserver(TCP_FAST_OPEN_STALLS_TIMEOUT, this, true);
         PrefsChanged(prefBranch, nullptr);
     }
 
     nsHttpChannelAuthProvider::InitializePrefs();
 
     mMisc.AssignLiteral("rv:" MOZILLA_UAVERSION);
 
     mCompatFirefox.AssignLiteral("Firefox/" MOZILLA_UAVERSION);
@@ -731,16 +741,30 @@ nsHttpHandler::IncrementFastOpenConsecut
         mFastOpenConsecutiveFailureCounter++;
         if (mFastOpenConsecutiveFailureCounter == mFastOpenConsecutiveFailureLimit) {
             LOG(("nsHttpHandler::IncrementFastOpenConsecutiveFailureCounter - "
                  "Fast open failed too many times"));
         }
     }
 }
 
+void
+nsHttpHandler::IncrementFastOpenStallsCounter()
+{
+  LOG(("nsHttpHandler::IncrementFastOpenStallsCounter - failed=%d "
+        "failure_limit=%d", mFastOpenStallsCounter, mFastOpenStallsLimit));
+  if (mFastOpenStallsCounter < mFastOpenStallsLimit) {
+    mFastOpenStallsCounter++;
+    if (mFastOpenStallsCounter == mFastOpenStallsLimit) {
+      LOG(("nsHttpHandler::IncrementFastOpenStallsCounter - "
+           "There are too many stalls involving TFO and TLS."));
+    }
+  }
+}
+
 nsresult
 nsHttpHandler::GetStreamConverterService(nsIStreamConverterService **result)
 {
     if (!mStreamConvSvc) {
         nsresult rv;
         nsCOMPtr<nsIStreamConverterService> service =
             do_GetService(NS_STREAMCONVERTERSERVICE_CONTRACTID, &rv);
         if (NS_FAILED(rv))
@@ -1924,16 +1948,46 @@ nsHttpHandler::PrefsChanged(nsIPrefBranc
         if (NS_SUCCEEDED(rv)) {
             if (val < 0) {
                 val = 0;
             }
             mFastOpenConsecutiveFailureLimit = val;
         }
     }
 
+    if (PREF_CHANGED(TCP_FAST_OPEN_STALLS_LIMIT)) {
+        rv = prefs->GetIntPref(TCP_FAST_OPEN_STALLS_LIMIT, &val);
+        if (NS_SUCCEEDED(rv)) {
+            if (val < 0) {
+                val = 0;
+            }
+            mFastOpenStallsLimit = val;
+        }
+    }
+
+    if (PREF_CHANGED(TCP_FAST_OPEN_STALLS_TIMEOUT)) {
+        rv = prefs->GetIntPref(TCP_FAST_OPEN_STALLS_TIMEOUT, &val);
+        if (NS_SUCCEEDED(rv)) {
+            if (val < 0) {
+                val = 0;
+            }
+            mFastOpenStallsTimeout = val;
+        }
+    }
+
+    if (PREF_CHANGED(TCP_FAST_OPEN_STALLS_IDLE)) {
+        rv = prefs->GetIntPref(TCP_FAST_OPEN_STALLS_IDLE, &val);
+        if (NS_SUCCEEDED(rv)) {
+            if (val < 0) {
+                val = 0;
+            }
+            mFastOpenStallsIdleTime = val;
+        }
+    }
+
     if (PREF_CHANGED(HTTP_PREF("spdy.hpack-default-buffer"))) {
         rv = prefs->GetIntPref(HTTP_PREF("spdy.default-hpack-buffer"), &val);
         if (NS_SUCCEEDED(rv)) {
             mDefaultHpackBuffer = val;
         }
     }
 
     // Enable HTTP response timeout if TCP Keepalives are disabled.
@@ -2270,18 +2324,20 @@ nsHttpHandler::Observe(nsISupports *subj
         }
 
         if (UseFastOpen()) {
             Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 0);
         } else if (!mFastOpenSupported) {
             Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 1);
         } else if (!mUseFastOpen) {
             Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 2);
+        } else if (mFastOpenConsecutiveFailureCounter >= mFastOpenConsecutiveFailureLimit) {
+            Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 3);
         } else {
-            Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 3);
+            Telemetry::Accumulate(Telemetry::TCP_FAST_OPEN_STATUS, 4);
         }
     } else if (!strcmp(topic, "profile-change-net-restore")) {
         // initialize connection manager
         rv = InitConnectionMgr();
         MOZ_ASSERT(NS_SUCCEEDED(rv));
     } else if (!strcmp(topic, "net:clear-active-logins")) {
         Unused << mAuthCache.ClearAll();
         Unused << mPrivateAuthCache.ClearAll();
--- a/netwerk/protocol/http/nsHttpHandler.h
+++ b/netwerk/protocol/http/nsHttpHandler.h
@@ -172,17 +172,18 @@ public:
     // same time used between successful keepalive probes.
     int32_t GetTCPKeepaliveLongLivedIdleTime() {
       return mTCPKeepaliveLongLivedIdleTimeS;
     }
 
     bool UseFastOpen()
     {
         return mUseFastOpen && mFastOpenSupported &&
-               mFastOpenConsecutiveFailureCounter < mFastOpenConsecutiveFailureLimit;
+               (mFastOpenStallsCounter < mFastOpenStallsLimit) &&
+               (mFastOpenConsecutiveFailureCounter < mFastOpenConsecutiveFailureLimit);
     }
     // If one of tcp connections return PR_NOT_TCP_SOCKET_ERROR while trying
     // fast open, it means that Fast Open is turned off so we will not try again
     // until a restart. This is only on Linux.
     // For windows 10 we can only check whether a version of windows support
     // Fast Open at run time, so if we get error PR_NOT_IMPLEMENTED_ERROR it
     // means that Fast Open is not supported and we will set mFastOpenSupported
     // to false.
@@ -190,16 +191,24 @@ public:
 
     void IncrementFastOpenConsecutiveFailureCounter();
 
     void ResetFastOpenConsecutiveFailureCounter()
     {
         mFastOpenConsecutiveFailureCounter = 0;
     }
 
+    void IncrementFastOpenStallsCounter();
+    uint32_t CheckIfConnectionIsStalledOnlyIfIdleForThisAmountOfSeconds() {
+        return mFastOpenStallsIdleTime;
+    }
+    uint32_t FastOpenStallsTimeout() {
+      return mFastOpenStallsTimeout;
+    }
+
     // returns the HTTP framing check level preference, as controlled with
     // network.http.enforce-framing.http1 and network.http.enforce-framing.soft
     FrameCheckLevel GetEnforceH1Framing() { return mEnforceH1Framing; }
 
     nsHttpAuthCache     *AuthCache(bool aPrivate) {
         return aPrivate ? &mPrivateAuthCache : &mAuthCache;
     }
     nsHttpConnectionMgr *ConnMgr()   { return mConnMgr; }
@@ -646,16 +655,20 @@ private:
 
     // The ratio for dispatching transactions from the focused window.
     float mFocusedWindowTransactionRatio;
 
     Atomic<bool, Relaxed> mUseFastOpen;
     Atomic<bool, Relaxed> mFastOpenSupported;
     uint32_t mFastOpenConsecutiveFailureLimit;
     uint32_t mFastOpenConsecutiveFailureCounter;
+    uint32_t mFastOpenStallsLimit;
+    uint32_t mFastOpenStallsCounter;
+    uint32_t mFastOpenStallsIdleTime;
+    uint32_t mFastOpenStallsTimeout;
 
     // If true, the transactions from active tab will be dispatched first.
     bool mActiveTabPriority;
 
 private:
     // For Rate Pacing Certain Network Events. Only assign this pointer on
     // socket thread.
     void MakeNewRequestTokenBucket();
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -2432,17 +2432,17 @@
     "alert_emails": ["necko@mozilla.com", "ddamjanovic@mozilla.com"],
     "bug_numbers": [1402879]
   },
   "TCP_FAST_OPEN_STATUS": {
     "record_in_processes": ["main", "content"],
     "expires_in_version": "61",
     "kind": "enumerated",
     "n_values": 8,
-    "description": "TCP Fast Open was: 0=enabled during the session, 1=not available or disabled in the os, 2=disabled by the pref, 3=disabled based on the too many connection failures.",
+    "description": "TCP Fast Open was: 0=enabled during the session, 1=not available or disabled in the os, 2=disabled by the pref, 3=disabled based on the too many connection failures, 4=disable based on too many http transaction stalls after a connection was idle.",
     "alert_emails": ["necko@mozilla.com", "ddamjanovic@mozilla.com"],
     "bug_numbers": [1390881]
   },
   "TLS_EARLY_DATA_NEGOTIATED": {
     "record_in_processes": ["main", "content"],
     "expires_in_version": "62",
     "kind": "enumerated",
     "n_values": 3,