Bug 1188435 - Make TLS 0RTT work with TLS Fast Open. r=mcmanus
authorDragana Damjanovic <dd.mozilla@gmail.com>
Thu, 04 May 2017 12:14:13 +0200
changeset 572779 033337b35af3ce70477bc53d70deaa0110e40d23
parent 572778 680a2f0d0b26b347a10786aa21e2bd980a2a5d16
child 572780 8adfceab18a55dafa1f0b0d8f57f877c91da3e0c
push id57195
push userbmo:rbarker@mozilla.com
push dateThu, 04 May 2017 20:08:56 +0000
reviewersmcmanus
bugs1188435
milestone55.0a1
Bug 1188435 - Make TLS 0RTT work with TLS Fast Open. r=mcmanus
netwerk/base/TCPFastOpen.h
netwerk/base/TCPFastOpenLayer.cpp
netwerk/base/TCPFastOpenLayer.h
netwerk/base/nsSocketTransport2.cpp
netwerk/protocol/http/nsHttpConnection.cpp
netwerk/protocol/http/nsHttpConnection.h
netwerk/protocol/http/nsHttpConnectionMgr.cpp
netwerk/protocol/http/nsHttpConnectionMgr.h
--- a/netwerk/base/TCPFastOpen.h
+++ b/netwerk/base/TCPFastOpen.h
@@ -27,17 +27,17 @@ class TCPFastOpen
 public:
 
   // Check if we have a transaction that is safe to be used with TFO.
   // Connections over TLS are always safe and some http requests (e.g. GET).
   virtual bool FastOpenEnabled() = 0;
   // To use TFO we need to have a transaction prepared, e.g. also have
   // nsHttpConnection ready. This functions is call by nsSocketTransport to
   // setup a connection.
-  virtual nsresult StartFastOpen() = 0;
+  virtual nsresult StartFastOpen(PRFileDesc *fd) = 0;
   // Inform nsHalfopenSocket whether a connection using TFO succeeded or not.
   // This will cancel the backup connection and in case of a failure rewind
   // the transaction.
   virtual void FastOpenConnected(nsresult error) = 0;
   virtual void FastOpenNotSupported() = 0;
 };
 
 }
--- a/netwerk/base/TCPFastOpenLayer.cpp
+++ b/netwerk/base/TCPFastOpenLayer.cpp
@@ -11,34 +11,68 @@
 
 namespace mozilla {
 namespace net {
 
 static PRDescIdentity sTCPFastOpenLayerIdentity;
 static PRIOMethods    sTCPFastOpenLayerMethods;
 static PRIOMethods   *sTCPFastOpenLayerMethodsPtr = nullptr;
 
+#define TFO_MAX_PACKET_SIZE_IPV4 1460
+#define TFO_MAX_PACKET_SIZE_IPV6 1440
+#define TFO_TLS_RECORD_HEADER_SIZE 22
+
+/**
+ *  For the TCP Fast Open it is necessary to send all data that can fit into the
+ *  first packet on a single sendto function call. Consecutive calls will not
+ *  have an effect. Therefore  TCPFastOpenLayer will collect some data before
+ *  calling sendto. Necko and nss will call PR_Write multiple times with small
+ *  amount of  data.
+ *  TCPFastOpenLayer has 4 states:
+ *    WAITING_FOR_CONNECT:
+ *      This is before connect is call. A call of recv, send or getpeername will
+ *      return PR_NOT_CONNECTED_ERROR. After connect is call the state transfers
+ *      into COLLECT_DATA_FOR_FIRST_PACKET.
+ *
+ *    COLLECT_DATA_FOR_FIRST_PACKET:
+ *      In this state all data received by send function calls will be stored in
+ *      a buffer. If transaction do not have any more data ready to be sent or
+ *      the buffer is full, TCPFastOpenFinish is call. TCPFastOpenFinish sends
+ *      the collected data using sendto function and the state transfers to
+ *      WAITING_FOR_CONNECTCONTINUE. If an error occurs during sendto, the error
+ *      is reported by the TCPFastOpenFinish return values. nsSocketTransfer is
+ *      the only caller of TCPFastOpenFinish; it knows how to interpreter these
+ *      errors.
+ *    WAITING_FOR_CONNECTCONTINUE:
+ *      connectcontinue transfers from this state to CONNECTED. Any other
+ *      function (e.g. send, recv) returns PR_WOULD_BLOCK_ERROR.
+ *    CONNECTED:
+ *      The size of mFirstPacketBuf is 1440/1460 (RFC7413 recomends that packet
+ *      does exceeds these sizes). SendTo does not have to consume all buffered
+ *      data and some data can be still in mFirstPacketBuf. Before sending any
+ *      new data we need to send the remaining buffered data.
+ **/
+
 class TCPFastOpenSecret
 {
 public:
   TCPFastOpenSecret()
     : mState(WAITING_FOR_CONNECT)
-    , mConnectResult(0)
-    , mFastOpenNotSupported(false)
+    , mFirstPacketBufLen(0)
   {}
 
   enum {
     CONNECTED,
     WAITING_FOR_CONNECTCONTINUE,
-    WAITING_FOR_FIRST_SEND,
+    COLLECT_DATA_FOR_FIRST_PACKET,
     WAITING_FOR_CONNECT
   } mState;
   PRNetAddr mAddr;
-  PRErrorCode mConnectResult;
-  bool mFastOpenNotSupported;
+  char mFirstPacketBuf[1460];
+  uint16_t mFirstPacketBufLen;
 };
 
 static PRStatus
 TCPFastOpenConnect(PRFileDesc *fd, const PRNetAddr *addr,
                    PRIntervalTime timeout)
 {
   MOZ_RELEASE_ASSERT(fd->identity == sTCPFastOpenLayerIdentity);
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
@@ -47,20 +81,19 @@ TCPFastOpenConnect(PRFileDesc *fd, const
 
   SOCKET_LOG(("TCPFastOpenConnect state=%d.\n", secret->mState));
 
   if (secret->mState != TCPFastOpenSecret::WAITING_FOR_CONNECT) {
     PR_SetError(PR_IS_CONNECTED_ERROR, 0);
     return PR_FAILURE;
   }
 
-  // Remember the address we will call PR_Sendto and for that we need
-  // the address.
+  // Remember the address. It will be used for sendto or connect later.
   memcpy(&secret->mAddr, addr, sizeof(secret->mAddr));
-  secret->mState = TCPFastOpenSecret::WAITING_FOR_FIRST_SEND;
+  secret->mState = TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET;
 
   return PR_SUCCESS;
 }
 
 static PRInt32
 TCPFastOpenSend(PRFileDesc *fd, const void *buf, PRInt32 amount,
                 PRIntn flags, PRIntervalTime timeout)
 {
@@ -68,62 +101,67 @@ TCPFastOpenSend(PRFileDesc *fd, const vo
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(fd->secret);
 
   SOCKET_LOG(("TCPFastOpenSend state=%d.\n", secret->mState));
 
   switch(secret->mState) {
   case TCPFastOpenSecret::CONNECTED:
+    // Before sending new data we need to drain the data collected during tfo.
+    if (secret->mFirstPacketBufLen) {
+      SOCKET_LOG(("TCPFastOpenSend - %d bytes to drain from "
+                  "mFirstPacketBufLen.\n",
+                  secret->mFirstPacketBufLen ));
+      PRInt32 rv = (fd->lower->methods->send)(fd->lower,
+                                              secret->mFirstPacketBuf,
+                                              secret->mFirstPacketBufLen,
+                                              0, // flags
+                                              PR_INTERVAL_NO_WAIT);
+      if (rv <= 0) {
+        return rv;
+      } else {
+        secret->mFirstPacketBufLen -= rv;
+        if (secret->mFirstPacketBufLen) {
+          memmove(secret->mFirstPacketBuf,
+                  secret->mFirstPacketBuf + rv,
+                  secret->mFirstPacketBufLen);
+
+          PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+          return PR_FAILURE;
+        } // if we drained the buffer we can fall through this checks and call
+          // send for the new data
+      }
+    }
+    SOCKET_LOG(("TCPFastOpenSend sending new data.\n"));
     return (fd->lower->methods->send)(fd->lower, buf, amount, flags, timeout);
   case TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE:
     PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
     return -1;
-  case TCPFastOpenSecret::WAITING_FOR_FIRST_SEND:
+  case TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET:
     {
-      PRInt32 rv = (fd->lower->methods->sendto)(fd->lower, buf, amount, flags,
-                                                &secret->mAddr, timeout);
+      int32_t toSend =
+        (secret->mAddr.raw.family == PR_AF_INET) ? TFO_MAX_PACKET_SIZE_IPV4
+                                              : TFO_MAX_PACKET_SIZE_IPV6;
+      MOZ_ASSERT(secret->mFirstPacketBufLen <= toSend);
+      toSend -= secret->mFirstPacketBufLen;
 
-      SOCKET_LOG(("TCPFastOpenSend - sendto result=%d.\n", rv));
-      if (rv > -1) {
-        secret->mState = TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE;
-        secret->mConnectResult = PR_IN_PROGRESS_ERROR;
-        return rv;
+      SOCKET_LOG(("TCPFastOpenSend: amount of data in the buffer=%d; the amount"
+                  " of additional data that can be stored=%d.\n",
+                  secret->mFirstPacketBufLen, toSend));
+
+      if (!toSend) {
+        PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
+        return -1;
       }
 
-      secret->mConnectResult = PR_GetError();
-      SOCKET_LOG(("TCPFastOpenSend - sendto error=%d.\n",
-                  secret->mConnectResult));
-
-      if (secret->mConnectResult == PR_IS_CONNECTED_ERROR) {
-        secret->mState = TCPFastOpenSecret::CONNECTED;
-
-      } else if (secret->mConnectResult == PR_IN_PROGRESS_ERROR) {
-        secret->mState = TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE;
-
-      } else if (secret->mConnectResult == PR_NOT_IMPLEMENTED_ERROR || // When a windows version does not support Fast Open it will return this error.
-                 secret->mConnectResult == PR_NOT_TCP_SOCKET_ERROR) { // SendTo will return PR_NOT_TCP_SOCKET_ERROR if TCP Fast Open is turned off on Linux.
-        // We can call connect again.
-        secret->mFastOpenNotSupported = true;
-        rv = (fd->lower->methods->connect)(fd->lower, &secret->mAddr, timeout);
-
-        if (rv == PR_SUCCESS) {
-          secret->mConnectResult = PR_IS_CONNECTED_ERROR;
-          secret->mState = TCPFastOpenSecret::CONNECTED;
-
-        } else {
-          secret->mConnectResult = PR_GetError();
-          secret->mState = TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE;
-
-        }
-      }
-
-      // Error will be picked up by TCPFastOpenConnectResult.
-      PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
-      return -1;
+      toSend = (toSend > amount) ? amount : toSend;
+      memcpy(secret->mFirstPacketBuf + secret->mFirstPacketBufLen, buf, toSend);
+      secret->mFirstPacketBufLen += toSend;
+      return toSend;
     }
   case TCPFastOpenSecret::WAITING_FOR_CONNECT:
     PR_SetError(PR_NOT_CONNECTED_ERROR, 0);
     return -1;
   }
   PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
   return PR_FAILURE;
 }
@@ -140,20 +178,44 @@ TCPFastOpenRecv(PRFileDesc *fd, void *bu
 {
   MOZ_RELEASE_ASSERT(fd->identity == sTCPFastOpenLayerIdentity);
 
   TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(fd->secret);
 
   PRInt32 rv = -1;
   switch(secret->mState) {
   case TCPFastOpenSecret::CONNECTED:
+
+    if (secret->mFirstPacketBufLen) {
+      // TLS will not call write before receiving data from a server, therefore
+      // we need to force sending buffered data even during recv call. Otherwise
+      // It can come to a deadlock (clients waits for response, but the request
+      // is sitting in mFirstPacketBufLen).
+      SOCKET_LOG(("TCPFastOpenRevc - %d bytes to drain from mFirstPacketBuf.\n",
+                  secret->mFirstPacketBufLen ));
+      PRInt32 rv = (fd->lower->methods->send)(fd->lower,
+                                              secret->mFirstPacketBuf,
+                                              secret->mFirstPacketBufLen,
+                                              0, // flags
+                                              PR_INTERVAL_NO_WAIT);
+      if (rv <= 0) {
+        return rv;
+      } else {
+        secret->mFirstPacketBufLen -= rv;
+        if (secret->mFirstPacketBufLen) {
+          memmove(secret->mFirstPacketBuf,
+                  secret->mFirstPacketBuf + rv,
+                  secret->mFirstPacketBufLen);
+        }
+      }
+    }
     rv = (fd->lower->methods->recv)(fd->lower, buf, amount, flags, timeout);
     break;
   case TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE:
-  case TCPFastOpenSecret::WAITING_FOR_FIRST_SEND:
+  case TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET:
     PR_SetError(PR_WOULD_BLOCK_ERROR, 0);
     break;
   case TCPFastOpenSecret::WAITING_FOR_CONNECT:
     PR_SetError(PR_NOT_CONNECTED_ERROR, 0);
   }
   return rv;
 }
 
@@ -171,27 +233,25 @@ TCPFastOpenConnectContinue(PRFileDesc *f
   TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(fd->secret);
 
   PRStatus rv = PR_FAILURE;
   switch(secret->mState) {
   case TCPFastOpenSecret::CONNECTED:
     rv = PR_SUCCESS;
     break;
   case TCPFastOpenSecret::WAITING_FOR_CONNECT:
-  case TCPFastOpenSecret::WAITING_FOR_FIRST_SEND:
+  case TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET:
     PR_SetError(PR_NOT_CONNECTED_ERROR, 0);
     rv = PR_FAILURE;
     break;
   case TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE:
     rv = (fd->lower->methods->connectcontinue)(fd->lower, out_flags);
 
     SOCKET_LOG(("TCPFastOpenConnectContinue result=%d.\n", rv));
-    if (rv == PR_SUCCESS) {
-      secret->mState = TCPFastOpenSecret::CONNECTED;
-    }
+    secret->mState = TCPFastOpenSecret::CONNECTED;
   }
   return rv;
 }
 
 static PRStatus
 TCPFastOpenClose(PRFileDesc *fd)
 {
   if (!fd) {
@@ -224,32 +284,47 @@ TCPFastOpenGetpeername (PRFileDesc *fd, 
     PR_SetError(PR_NOT_CONNECTED_ERROR, 0);
     return PR_FAILURE;
   }
 
   memcpy(addr, &secret->mAddr, sizeof(secret->mAddr));
   return PR_SUCCESS;
 }
 
+static PRInt16
+TCPFastOpenPoll(PRFileDesc *fd, PRInt16 how_flags, PRInt16 *p_out_flags)
+{
+  MOZ_RELEASE_ASSERT(fd);
+  MOZ_RELEASE_ASSERT(fd->identity == sTCPFastOpenLayerIdentity);
+
+  TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(fd->secret);
+  if (secret->mFirstPacketBufLen) {
+    how_flags |= PR_POLL_WRITE;
+  }
+
+  return fd->lower->methods->poll(fd->lower, how_flags, p_out_flags);
+}
+
 nsresult
 AttachTCPFastOpenIOLayer(PRFileDesc *fd)
 {
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   if (!sTCPFastOpenLayerMethodsPtr) {
     sTCPFastOpenLayerIdentity = PR_GetUniqueIdentity("TCPFastOpen Layer");
     sTCPFastOpenLayerMethods = *PR_GetDefaultIOMethods();
     sTCPFastOpenLayerMethods.connect = TCPFastOpenConnect;
     sTCPFastOpenLayerMethods.send = TCPFastOpenSend;
     sTCPFastOpenLayerMethods.write = TCPFastOpenWrite;
     sTCPFastOpenLayerMethods.recv = TCPFastOpenRecv;
     sTCPFastOpenLayerMethods.read = TCPFastOpenRead;
     sTCPFastOpenLayerMethods.connectcontinue = TCPFastOpenConnectContinue;
     sTCPFastOpenLayerMethods.close = TCPFastOpenClose;
     sTCPFastOpenLayerMethods.getpeername = TCPFastOpenGetpeername;
+    sTCPFastOpenLayerMethods.poll = TCPFastOpenPoll;
     sTCPFastOpenLayerMethodsPtr = &sTCPFastOpenLayerMethods;
   }
 
   PRFileDesc *layer = PR_CreateIOLayerStub(sTCPFastOpenLayerIdentity,
                                            sTCPFastOpenLayerMethodsPtr);
 
   if (!layer) {
     return NS_ERROR_FAILURE;
@@ -265,39 +340,120 @@ AttachTCPFastOpenIOLayer(PRFileDesc *fd)
     delete secret;
     PR_DELETE(layer);
     return NS_ERROR_FAILURE;
   }
   return NS_OK;
 }
 
 void
-TCPFastOpenConnectResult(PRFileDesc * fd, PRErrorCode *err,
-                         bool *fastOpenNotSupported)
+TCPFastOpenFinish(PRFileDesc * fd, PRErrorCode *err,
+                  bool *fastOpenNotSupported)
 {
   PRFileDesc *tfoFd = PR_GetIdentitiesLayer(fd, sTCPFastOpenLayerIdentity);
   MOZ_RELEASE_ASSERT(tfoFd);
   MOZ_ASSERT(OnSocketThread(), "not on socket thread");
 
   TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(tfoFd->secret);
 
-  MOZ_ASSERT(secret->mState != TCPFastOpenSecret::WAITING_FOR_CONNECT);
+  MOZ_ASSERT(secret->mState == TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET);
 
-  *fastOpenNotSupported = secret->mFastOpenNotSupported;
+  *fastOpenNotSupported = false;
+  PRErrorCode result = 0;
 
-  if (secret->mState == TCPFastOpenSecret::WAITING_FOR_FIRST_SEND) {
-    // Because of the way our HttpTransaction dispatch work, it can happened
-    // that connect is not called.
+  if (!secret->mFirstPacketBufLen) {
+    // Because of the way our nsHttpTransaction dispatch work, it can happened
+    // that data has not been written into the socket.
+    // In this case we can just call connect.
     PRInt32 rv = (tfoFd->lower->methods->connect)(tfoFd->lower, &secret->mAddr,
-                                               PR_INTERVAL_NO_WAIT);
+                                                  PR_INTERVAL_NO_WAIT);
     if (rv == PR_SUCCESS) {
-      secret->mConnectResult = PR_IS_CONNECTED_ERROR;
-      secret->mState = TCPFastOpenSecret::CONNECTED;
+      result = PR_IS_CONNECTED_ERROR;
     } else {
-      secret->mConnectResult = PR_GetError();
-      secret->mState = TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE;
+      result = PR_GetError();
+    }
+  } else {
+    // We have some data ready in the buffer we will send it with the syn
+    // packet.
+    PRInt32 rv = (tfoFd->lower->methods->sendto)(tfoFd->lower,
+                                                 secret->mFirstPacketBuf,
+                                                 secret->mFirstPacketBufLen,
+                                                 0, //flags
+                                                 &secret->mAddr,
+                                                 PR_INTERVAL_NO_WAIT);
+
+    SOCKET_LOG(("TCPFastOpenFinish - sendto result=%d.\n", rv));
+    if (rv > 0) {
+      result = PR_IN_PROGRESS_ERROR;
+      secret->mFirstPacketBufLen -= rv;
+      if (secret->mFirstPacketBufLen) {
+        memmove(secret->mFirstPacketBuf,
+                secret->mFirstPacketBuf + rv,
+                secret->mFirstPacketBufLen);
+      }
+    } else {
+      result = PR_GetError();
+      SOCKET_LOG(("TCPFastOpenFinish - sendto error=%d.\n", result));
+
+      if (result == PR_NOT_IMPLEMENTED_ERROR || // When a windows version does not support Fast Open it will return this error.
+          result == PR_NOT_TCP_SOCKET_ERROR) { // SendTo will return PR_NOT_TCP_SOCKET_ERROR if TCP Fast Open is turned off on Linux.
+        // We can call connect again.
+        *fastOpenNotSupported = true;
+        rv = (tfoFd->lower->methods->connect)(tfoFd->lower, &secret->mAddr,
+                                              PR_INTERVAL_NO_WAIT);
+
+        if (rv == PR_SUCCESS) {
+          result = PR_IS_CONNECTED_ERROR;
+        } else {
+          result = PR_GetError();
+        }
+      }
     }
   }
-  *err = secret->mConnectResult;
+
+  if (result == PR_IN_PROGRESS_ERROR) {
+    secret->mState = TCPFastOpenSecret::WAITING_FOR_CONNECTCONTINUE;
+  } else {
+    // If the error is not PR_IN_PROGRESS_ERROR, change the state to CONNECT so
+    // that recv/send can perform recv/send on the next lower layer and pick up
+    // the real error. This is really important!
+    // The result can also be PR_IS_CONNECTED_ERROR, that should change the
+    // state to CONNECT anyway.
+    secret->mState = TCPFastOpenSecret::CONNECTED;
+  }
+  *err = result;
+}
+
+/* This function returns the size of the remaining free space in the
+ * first_packet_buffer. This will be used by transactions with a tls layer. For
+ * other transactions it is not necessary. The tls transactions make a tls
+ * record before writing to this layer and if the record is too big the part
+ * that does not have place in the mFirstPacketBuf will be saved on the tls
+ * layer. During TFO we cannot send more than TFO_MAX_PACKET_SIZE_IPV4/6 bytes,
+ * so if we have a big tls record, this record is encrypted with 0RTT key,
+ * tls-early-data can be rejected and than we still need to send the rest of the
+ * record.
+ */
+int32_t
+TCPFastOpenGetBufferSizeLeft(PRFileDesc *fd)
+{
+  PRFileDesc *tfoFd = PR_GetIdentitiesLayer(fd, sTCPFastOpenLayerIdentity);
+  MOZ_RELEASE_ASSERT(tfoFd);
+  MOZ_ASSERT(OnSocketThread(), "not on socket thread");
+
+  TCPFastOpenSecret *secret = reinterpret_cast<TCPFastOpenSecret *>(tfoFd->secret);
+
+  MOZ_ASSERT(secret->mState == TCPFastOpenSecret::COLLECT_DATA_FOR_FIRST_PACKET);
+
+  int32_t sizeLeft =
+    (secret->mAddr.raw.family == PR_AF_INET) ? TFO_MAX_PACKET_SIZE_IPV4
+                                             : TFO_MAX_PACKET_SIZE_IPV6;
+  MOZ_ASSERT(secret->mFirstPacketBufLen <= sizeLeft);
+  sizeLeft -= secret->mFirstPacketBufLen;
+
+  SOCKET_LOG(("TCPFastOpenGetBufferSizeLeft=%d.\n", sizeLeft));
+
+  return (sizeLeft > TFO_TLS_RECORD_HEADER_SIZE) ?
+    sizeLeft - TFO_TLS_RECORD_HEADER_SIZE : 0;
 }
 
 }
 }
--- a/netwerk/base/TCPFastOpenLayer.h
+++ b/netwerk/base/TCPFastOpenLayer.h
@@ -7,18 +7,25 @@
 #ifndef TCPFastOpenLayer_h__
 #define TCPFastOpenLayer_h__
 
 #include "prerror.h"
 
 namespace mozilla {
 namespace net {
 
+/**
+ * This layer must be placed just above PR-tcp socket, i.e. it must be under
+ * nss layer.
+ * At the beginning of TCPFastOpenLayer.cpp there is explanation what this
+ * layer do.
+ **/
 nsresult AttachTCPFastOpenIOLayer(PRFileDesc *fd);
 
 // Get the result of TCP Fast Open.
-void TCPFastOpenConnectResult(PRFileDesc *fd, PRErrorCode *err,
-                              bool *fastOpenNotSupported);
+void TCPFastOpenFinish(PRFileDesc *fd, PRErrorCode *err,
+                       bool *fastOpenNotSupported);
 
+int32_t TCPFastOpenGetBufferSizeLeft(PRFileDesc *fd);
 }
 }
 
 #endif // TCPFastOpenLayer_h__
--- a/netwerk/base/nsSocketTransport2.cpp
+++ b/netwerk/base/nsSocketTransport2.cpp
@@ -1513,22 +1513,22 @@ nsSocketTransport::InitiateSocket()
     status = PR_Connect(fd, &prAddr, NS_SOCKET_CONNECT_TIMEOUT);
     PRErrorCode code = PR_GetError();
     if ((status == PR_SUCCESS) && tfo) {
         {
             MutexAutoLock lock(mLock);
             mFDFastOpenInProgress = true;
         }
         SOCKET_LOG(("Using TCP Fast Open."));
-        rv = mFastOpenCallback->StartFastOpen();
+        rv = mFastOpenCallback->StartFastOpen(fd);
         status = PR_FAILURE;
         connectCalled = false;
         bool fastOpenNotSupported = false;
 
-        TCPFastOpenConnectResult(fd, &code, &fastOpenNotSupported);
+        TCPFastOpenFinish(fd, &code, &fastOpenNotSupported);
         SOCKET_LOG(("called StartFastOpen - code=%d; fastOpen is %s "
                     "supported.\n", code,
                     fastOpenNotSupported ? "not" : ""));
 
         if (fastOpenNotSupported) {
           // When TCP_FastOpen is turned off on the local host
           // SendTo will return PR_NOT_TCP_SOCKET_ERROR. This is only
           // on Linux.
--- a/netwerk/protocol/http/nsHttpConnection.cpp
+++ b/netwerk/protocol/http/nsHttpConnection.cpp
@@ -30,16 +30,17 @@
 #include "nsISSLSocketControl.h"
 #include "nsISupportsPriority.h"
 #include "nsPreloadedStream.h"
 #include "nsProxyRelease.h"
 #include "nsSocketTransport2.h"
 #include "nsStringStream.h"
 #include "sslt.h"
 #include "TunnelUtils.h"
+#include "TCPFastOpenLayer.h"
 
 namespace mozilla {
 namespace net {
 
 //-----------------------------------------------------------------------------
 // nsHttpConnection <public>
 //-----------------------------------------------------------------------------
 
@@ -82,17 +83,17 @@ nsHttpConnection::nsHttpConnection()
     , mForceSendPending(false)
     , m0RTTChecked(false)
     , mWaitingFor0RTTResponse(false)
     , mContentBytesWritten0RTT(0)
     , mEarlyDataNegotiated(false)
     , mDid0RTTSpdy(false)
     , mResponseThrottled(false)
     , mResumeRecvOnUnthrottle(false)
-    , mFastOpen(false)
+    , mFastOpen(nullptr)
 {
     LOG(("Creating nsHttpConnection @%p\n", this));
 
     // the default timeout is for when this connection has not yet processed a
     // transaction
     static const PRIntervalTime k5Sec = PR_SecondsToInterval(5);
     mIdleTimeout =
         (k5Sec < gHttpHandler->IdleTimeout()) ? k5Sec : gHttpHandler->IdleTimeout();
@@ -378,19 +379,17 @@ nsHttpConnection::EnsureNPNComplete(nsre
     if (!m0RTTChecked) {
         // We reuse m0RTTChecked. We want to send this status only once.
         mTransaction->OnTransportStatus(mSocketTransport,
                                         NS_NET_STATUS_TLS_HANDSHAKE_STARTING,
                                         0);
     }
 
     rv = ssl->GetNegotiatedNPN(negotiatedNPN);
-    // Fast Open does not work well with TLS Early Data. TODO: dragana
-    // fix this.
-    if (!mFastOpen && !m0RTTChecked && (rv == NS_ERROR_NOT_CONNECTED) &&
+    if (!m0RTTChecked && (rv == NS_ERROR_NOT_CONNECTED) &&
         !mConnInfo->UsingProxy()) {
         // There is no ALPN info (yet!). We need to consider doing 0RTT. We
         // will do so if there is ALPN information from a previous session
         // (AlpnEarlySelection), we are using HTTP/1, and the request data can
         // be safely retried.
         m0RTTChecked = true;
         nsresult rvEarlyAlpn = ssl->GetAlpnEarlySelection(mEarlyNegotiatedALPN);
         if (NS_FAILED(rvEarlyAlpn)) {
@@ -439,25 +438,34 @@ nsHttpConnection::EnsureNPNComplete(nsre
                 Start0RTTSpdy(info->Version[infoIndex]);
             }
             mEarlyDataNegotiated = true;
         }
     }
 
     if (rv == NS_ERROR_NOT_CONNECTED) {
         if (mWaitingFor0RTTResponse) {
-            aOut0RTTWriteHandshakeValue = mTransaction->ReadSegments(this,
-                nsIOService::gDefaultSegmentSize, &aOut0RTTBytesWritten);
-            if (NS_FAILED(aOut0RTTWriteHandshakeValue) &&
-                aOut0RTTWriteHandshakeValue != NS_BASE_STREAM_WOULD_BLOCK) {
-                goto npnComplete;
+            int32_t segmentSize = nsIOService::gDefaultSegmentSize;
+            if (mFastOpen) {
+                segmentSize = TCPFastOpenGetBufferSizeLeft(mFastOpen);
+                LOG(("nsHttpConnection::EnsureNPNComplete [this=%p] - It is "
+                     "fast open and the first packet has %d byte to fill.",
+                     this, segmentSize));
             }
-            LOG(("nsHttpConnection::EnsureNPNComplete [this=%p] - written %d "
-                 "bytes during 0RTT", this, aOut0RTTBytesWritten));
-            mContentBytesWritten0RTT += aOut0RTTBytesWritten;
+            if (segmentSize) {
+                aOut0RTTWriteHandshakeValue = mTransaction->ReadSegments(this,
+                    segmentSize, &aOut0RTTBytesWritten);
+                if (NS_FAILED(aOut0RTTWriteHandshakeValue) &&
+                    aOut0RTTWriteHandshakeValue != NS_BASE_STREAM_WOULD_BLOCK) {
+                    goto npnComplete;
+                }
+                LOG(("nsHttpConnection::EnsureNPNComplete [this=%p] - written %d "
+                     "bytes during 0RTT", this, aOut0RTTBytesWritten));
+                mContentBytesWritten0RTT += aOut0RTTBytesWritten;
+            }
         }
 
         rv = ssl->DriveHandshake();
         if (NS_FAILED(rv) && rv != NS_BASE_STREAM_WOULD_BLOCK) {
             goto npnComplete;
         }
 
         return false;
--- a/netwerk/protocol/http/nsHttpConnection.h
+++ b/netwerk/protocol/http/nsHttpConnection.h
@@ -78,17 +78,17 @@ public:
                                nsIInterfaceRequestor *, PRIntervalTime);
 
     // Activate causes the given transaction to be processed on this
     // connection.  It fails if there is already an existing transaction unless
     // a multiplexing protocol such as SPDY is being used
     MOZ_MUST_USE nsresult Activate(nsAHttpTransaction *, uint32_t caps,
                                    int32_t pri);
 
-    void SetFastOpen(bool aFastOpen) { mFastOpen = aFastOpen; }
+    void SetFastOpen(PRFileDesc *aFastOpen) { mFastOpen = aFastOpen; }
     // Close this connection and return the transaction. The transaction is
     // restarted as well. This will only happened before connection is
     // connected.
     nsAHttpTransaction * CloseConnectionFastOpenTakesTooLongOrError();
 
     // Close the underlying socket transport.
     void Close(nsresult reason, bool aIsShutdown = false);
 
@@ -394,17 +394,17 @@ private:
 
     // Reflects throttling request, effects if we resume read from the socket.
     // Accessed only on the socket thread.
     bool                           mResponseThrottled;
     // A read from the socket was requested while we where throttled, means
     // to ResumeRecv() when untrottled again. Only accessed on the socket thread.
     bool                           mResumeRecvOnUnthrottle;
 
-    bool                           mFastOpen;
+    PRFileDesc                    *mFastOpen;
 };
 
 NS_DEFINE_STATIC_IID_ACCESSOR(nsHttpConnection, NS_HTTPCONNECTION_IID)
 
 } // namespace net
 } // namespace mozilla
 
 #endif // nsHttpConnection_h__
--- a/netwerk/protocol/http/nsHttpConnectionMgr.cpp
+++ b/netwerk/protocol/http/nsHttpConnectionMgr.cpp
@@ -3451,17 +3451,17 @@ nsHalfOpenSocket::OnOutputStreamReady(ns
             }
         }
         if (mEnt->mUseFastOpen) {
             gHttpHandler->IncrementFastOpenConsecutiveFailureCounter();
             mEnt->mUseFastOpen = false;
         }
     }
 
-    return SetupConn(out, false);
+    return SetupConn(out, nullptr);
 }
 
 bool
 nsHttpConnectionMgr::
 nsHalfOpenSocket::FastOpenEnabled()
 {
     LOG(("nsHalfOpenSocket::FastOpenEnabled [this=%p]\n", this));
 
@@ -3500,30 +3500,31 @@ nsHalfOpenSocket::FastOpenEnabled()
         }
     }
 
     return true;
 }
 
 nsresult
 nsHttpConnectionMgr::
-nsHalfOpenSocket::StartFastOpen()
+nsHalfOpenSocket::StartFastOpen(PRFileDesc *fd)
 {
+    MOZ_ASSERT(fd);
     MOZ_ASSERT(mStreamOut);
     MOZ_ASSERT(mEnt && !mBackupTransport);
     mUsingFastOpen = true;
     if (mEnt && !mBackupTransport && !mSynTimer) {
         // For Fast Open we will setup backup timer also for NullTransaction.
         // So maybe it is not set and we need to set it here.
         SetupBackupTimer();
     }
     mStreamOut->AsyncWait(nullptr, 0, 0, nullptr);
     mSocketTransport->SetEventSink(nullptr, nullptr);
     gHttpHandler->ConnMgr()->RecvdConnect();
-    return SetupConn(mStreamOut, true);
+    return SetupConn(mStreamOut, fd);
 }
 
 void
 nsHttpConnectionMgr::
 nsHalfOpenSocket::FastOpenConnected(nsresult aError)
 {
     RefPtr<nsHalfOpenSocket> deleteProtector(this);
     CancelBackupTimer();
@@ -3563,17 +3564,17 @@ nsHalfOpenSocket::FastOpenNotSupported()
   mConnectionNegotiatingFastOpen = nullptr;
   mSocketTransport = nullptr;
   gHttpHandler->SetFastOpenNotSupported();
 }
 
 nsresult
 nsHttpConnectionMgr::
 nsHalfOpenSocket::SetupConn(nsIAsyncOutputStream *out,
-                            bool aFastOpen)
+                            PRFileDesc *aFastOpen)
 {
     MOZ_ASSERT(!aFastOpen || (out == mStreamOut));
     // assign the new socket to the http connection
     RefPtr<nsHttpConnection> conn = new nsHttpConnection();
     LOG(("nsHalfOpenSocket::SetupConn "
          "Created new nshttpconnection %p\n", conn.get()));
 
     // Some capabilities are needed before a transaciton actually gets
@@ -3722,17 +3723,17 @@ nsHalfOpenSocket::SetupConn(nsIAsyncOutp
             }
         }
     }
     if (aFastOpen) {
         // If it is fast open create a new tranaction for backup stream.
         mTransaction = new NullHttpTransaction(mEnt->mConnInfo,
                                                callbacks, mCaps);
 
-        conn->SetFastOpen(false);
+        conn->SetFastOpen(nullptr);
         mConnectionNegotiatingFastOpen = conn;
     }
 
     // If this halfOpenConn was speculative, but at the ende the conn got a
     // non-null transaction than this halfOpen is not speculative anymore!
     if (conn->Transaction() && conn->Transaction()->IsNullTransaction()) {
         mSpeculative = false;
     }
--- a/netwerk/protocol/http/nsHttpConnectionMgr.h
+++ b/netwerk/protocol/http/nsHttpConnectionMgr.h
@@ -370,22 +370,22 @@ private:
         bool HasConnected() { return mHasConnected; }
 
         void PrintDiagnostics(nsCString &log);
 
         bool Claim();
         void Unclaim();
 
         bool FastOpenEnabled() override;
-        nsresult StartFastOpen() override;
+        nsresult StartFastOpen(PRFileDesc *) override;
         void FastOpenConnected(nsresult) override;
         void FastOpenNotSupported() override;
     private:
         nsresult SetupConn(nsIAsyncOutputStream *out,
-                           bool aFastOpen);
+                           PRFileDesc *aFastOpen);
 
         // To find out whether |mTransaction| is still in the connection entry's
         // pending queue. If the transaction is found and |removeWhenFound| is
         // true, the transaction will be removed from the pending queue.
         already_AddRefed<PendingTransactionInfo>
         FindTransactionHelper(bool removeWhenFound);
 
         nsConnectionEntry              *mEnt;