Bug 1404061, Permanent version of TCP Fast Open, Cancel overlappedIO before closing it. r=mcmanus, r=:kaie: NSPR_4_18_BETA1
authorKai Engert <kaie@kuix.de>
Fri, 24 Nov 2017 13:39:12 +0100
changeset 4764 1572f7b21403df4bbbbac19a00da12892ce47e87
parent 4763 680830e46dc6b6bb4b0f869ce6d11d4c559b6c2a
child 4765 2b5ef04c14838acd59f640c6952d412a0716b8ea
push id278
push userkaie@kuix.de
push dateFri, 24 Nov 2017 12:38:50 +0000
reviewersmcmanus
bugs1404061
Bug 1404061, Permanent version of TCP Fast Open, Cancel overlappedIO before closing it. r=mcmanus, r=:kaie:
pr/include/private/pprio.h
pr/include/private/primpl.h
pr/src/io/prio.c
pr/src/io/prsocket.c
pr/src/md/windows/w95sock.c
pr/src/md/windows/w95thred.c
pr/src/pthreads/ptio.c
pr/src/pthreads/ptthread.c
--- a/pr/include/private/pprio.h
+++ b/pr/include/private/pprio.h
@@ -232,25 +232,11 @@ NSPR_API(void) PR_NTFast_UpdateAcceptCon
 ** DESCRIPTION:
 **    Cancel IO operations on fd.
 */
 NSPR_API(PRStatus) PR_NT_CancelIo(PRFileDesc *fd);
 
 
 #endif /* WIN32 */
 
-/* FUNCTION: PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle
-** DESCRIPTION:
-** This function will be available only in nspr version 4.17
-** This functionality is only available on windows. Some windows operation use
-** asynchronous (call overlapped) io. One of them is ConnectEx. NSPR uses
-** ConnectEx for enabling TCP Fast Open.
-** This function returns an OVERLAPPED structure associated with ConnectEx call.
-** If ConnectEx has not been called or the io has already finished, the
-** function will return PR_INVALID_ARGUMENT_ERROR.
-** PRFileDesc continues to be owner of the structure and the structure must not
-** be destroyed.
-*/
-NSPR_API(PRStatus) PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol);
-
 PR_END_EXTERN_C
 
 #endif /* pprio_h___ */
--- a/pr/include/private/primpl.h
+++ b/pr/include/private/primpl.h
@@ -2160,11 +2160,23 @@ typedef struct _ConnectListNode {
 } ConnectListNode;
 
 extern ConnectListNode connectList[64];
 
 extern PRUint32 connectCount;
 
 #endif /* XP_BEOS */
 
+#if defined(_WIN64) && defined(WIN95)
+typedef struct _PRFileDescList {
+  PRFileDesc *fd;
+  struct _PRFileDescList *next;
+} PRFileDescList;
+
+extern PRLock *_fd_waiting_for_overlapped_done_lock;
+extern PRFileDescList *_fd_waiting_for_overlapped_done;
+extern void CheckOverlappedPendingSocketsAreDone();
+#endif
+
+
 PR_END_EXTERN_C
 
 #endif /* primpl_h___ */
--- a/pr/src/io/prio.c
+++ b/pr/src/io/prio.c
@@ -132,21 +132,92 @@ PR_IMPLEMENT(PRFileDesc*) PR_AllocFileDe
 }
 
 PR_IMPLEMENT(void) PR_FreeFileDesc(PRFileDesc *fd)
 {
     PR_ASSERT(fd);
     _PR_Putfd(fd);
 }
 
+#if defined(_WIN64) && defined(WIN95)
+
+PRFileDescList *_fd_waiting_for_overlapped_done = NULL;
+PRLock *_fd_waiting_for_overlapped_done_lock = NULL;
+
+void CheckOverlappedPendingSocketsAreDone()
+{
+  if (!_fd_waiting_for_overlapped_done_lock ||
+      !_fd_waiting_for_overlapped_done) {
+    return;
+  }
+
+  PR_Lock(_fd_waiting_for_overlapped_done_lock);
+
+  PRFileDescList *cur = _fd_waiting_for_overlapped_done;
+  PRFileDescList *previous = NULL;
+  while (cur) {
+    PR_ASSERT(cur->fd->secret->overlappedActive);
+    PRFileDesc *fd = cur->fd;
+    DWORD rvSent;
+    if (GetOverlappedResult((HANDLE)fd->secret->md.osfd, &fd->secret->ol, &rvSent, FALSE) == TRUE) {
+      fd->secret->overlappedActive = PR_FALSE;
+      PR_LOG(_pr_io_lm, PR_LOG_MIN,
+             ("CheckOverlappedPendingSocketsAreDone GetOverlappedResult succeeded\n"));
+    } else {
+      DWORD err = WSAGetLastError();
+      PR_LOG(_pr_io_lm, PR_LOG_MIN,
+             ("CheckOverlappedPendingSocketsAreDone GetOverlappedResult failed %d\n", err));
+      if (err != ERROR_IO_INCOMPLETE) {
+        fd->secret->overlappedActive = PR_FALSE;
+      }
+    }
+
+    if (!fd->secret->overlappedActive) {
+
+      _PR_MD_CLOSE_SOCKET(fd->secret->md.osfd);
+      fd->secret->state = _PR_FILEDESC_CLOSED;
+#ifdef _PR_HAVE_PEEK_BUFFER
+      if (fd->secret->peekBuffer) {
+        PR_ASSERT(fd->secret->peekBufSize > 0);
+        PR_DELETE(fd->secret->peekBuffer);
+        fd->secret->peekBufSize = 0;
+        fd->secret->peekBytes = 0;
+      }
+#endif
+
+      PR_FreeFileDesc(fd);
+
+      if (previous) {
+        previous->next = cur->next;
+      } else {
+        _fd_waiting_for_overlapped_done = cur->next;
+      }
+      PRFileDescList *del = cur;
+      cur = cur->next;
+      PR_Free(del);
+    } else {
+      previous = cur;
+      cur = cur->next;
+    }
+  }
+
+  PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+}
+#endif
+
 /*
 ** Wait for some i/o to finish on one or more more poll descriptors.
 */
 PR_IMPLEMENT(PRInt32) PR_Poll(PRPollDesc *pds, PRIntn npds, PRIntervalTime timeout)
 {
+#if defined(_WIN64) && defined(WIN95)
+  // For each iteration check if TFO overlapped IOs are down.
+  CheckOverlappedPendingSocketsAreDone();
+#endif
+
 	return(_PR_MD_PR_POLL(pds, npds, timeout));
 }
 
 /*
 ** Set the inheritance attribute of a file descriptor.
 */
 PR_IMPLEMENT(PRStatus) PR_SetFDInheritable(
     PRFileDesc *fd,
--- a/pr/src/io/prsocket.c
+++ b/pr/src/io/prsocket.c
@@ -732,16 +732,66 @@ static PRStatus PR_CALLBACK SocketClose(
 	if (!fd || !fd->secret
 			|| (fd->secret->state != _PR_FILEDESC_OPEN
 			&& fd->secret->state != _PR_FILEDESC_CLOSED)) {
 		PR_SetError(PR_BAD_DESCRIPTOR_ERROR, 0);
 		return PR_FAILURE;
 	}
 
 	if (fd->secret->state == _PR_FILEDESC_OPEN) {
+#if defined(_WIN64) && defined(WIN95)
+    /* TCP Fast Open on Windows must use ConnectEx, which uses overlapped
+     * input/output. Before closing such a socket we must cancelIO.
+     */
+    if (fd->secret->overlappedActive) {
+      PR_ASSERT(fd->secret->nonblocking);
+      if (CancelIo((HANDLE) fd->secret->md.osfd) == TRUE) {
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose - CancelIo succeeded\n"));
+      } else {
+        DWORD err = WSAGetLastError();
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose - CancelIo failed err=%x\n", err));
+      }
+
+      DWORD rvSent;
+      if (GetOverlappedResult((HANDLE)fd->secret->md.osfd, &fd->secret->ol, &rvSent, FALSE) == TRUE) {
+        fd->secret->overlappedActive = PR_FALSE;
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose GetOverlappedResult succeeded\n"));
+      } else {
+        DWORD err = WSAGetLastError();
+        PR_LOG(_pr_io_lm, PR_LOG_MIN,
+               ("SocketClose GetOverlappedResult failed %d\n", err));
+        if (err != ERROR_IO_INCOMPLETE) {
+          _PR_MD_MAP_CONNECT_ERROR(err);
+          fd->secret->overlappedActive = PR_FALSE;
+        }
+      }
+    }
+
+    if (fd->secret->overlappedActive &&
+        _fd_waiting_for_overlapped_done_lock) {
+      // Put osfd into the list to be checked later.
+      PRFileDescList *forWaiting = PR_NEW(PRFileDescList);
+      if (!forWaiting) {
+        PR_SetError(PR_OUT_OF_MEMORY_ERROR, 0);
+        return PR_FAILURE;
+      }
+      forWaiting->fd = fd;
+
+      PR_Lock(_fd_waiting_for_overlapped_done_lock);
+      forWaiting->next = _fd_waiting_for_overlapped_done;
+      _fd_waiting_for_overlapped_done = forWaiting;
+      PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+
+      return PR_SUCCESS;
+    }
+#endif
+
 		if (_PR_MD_CLOSE_SOCKET(fd->secret->md.osfd) < 0) {
 			return PR_FAILURE;
 		}
 		fd->secret->state = _PR_FILEDESC_CLOSED;
 	}
 
 #ifdef _PR_HAVE_PEEK_BUFFER
 	if (fd->secret->peekBuffer) {
@@ -1679,43 +1729,16 @@ PR_FileDesc2NativeHandle(PRFileDesc *fd)
 
 PR_IMPLEMENT(void)
 PR_ChangeFileDescNativeHandle(PRFileDesc *fd, PROsfd handle)
 {
 	if (fd)
 		fd->secret->md.osfd = handle;
 }
 
-/* Expose OVERLAPPED if present. OVERLAPPED is implemented only on WIN95. */
-PR_IMPLEMENT(PRStatus)
-PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol)
-{
-#if defined(_WIN64) && defined(WIN95)
-    *ol = NULL;
-    if (fd) {
-        fd = PR_GetIdentitiesLayer(fd, PR_NSPR_IO_LAYER);
-    }
-    if (!fd) {
-        PR_SetError(PR_INVALID_ARGUMENT_ERROR, 0);
-        return PR_FAILURE;
-    }
-
-    if (!fd->secret->overlappedActive) {
-        PR_SetError(PR_INVALID_ARGUMENT_ERROR, 0);
-        return PR_FAILURE;
-    }
-
-    *ol = &fd->secret->ol;
-    return PR_SUCCESS;
-#else
-    PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
-    return PR_FAILURE;
-#endif
-}
-
 /*
 ** Select compatibility
 **
 */
 
 PR_IMPLEMENT(void) PR_FD_ZERO(PR_fd_set *set)
 {
 	memset(set, 0, sizeof(PR_fd_set));
--- a/pr/src/md/windows/w95sock.c
+++ b/pr/src/md/windows/w95sock.c
@@ -377,16 +377,21 @@ static PRStatus PR_CALLBACK _pr_set_conn
     rc = closesocket(sock);
     return PR_SUCCESS;
 }
 
 PRInt32
 _PR_MD_TCPSENDTO(PRFileDesc *fd, const void *buf, PRInt32 amount, PRIntn flags,
                  const PRNetAddr *addr, PRUint32 addrlen, PRIntervalTime timeout)
 {
+    if (!_fd_waiting_for_overlapped_done_lock) {
+        PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
+        return PR_FAILURE;
+    }
+
     if (PR_CallOnce(&_pr_has_connectex_once, _pr_set_connectex) != PR_SUCCESS) {
         PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
         return PR_FAILURE;
     }
 
     if (_pr_win_connectex == NULL) {
         PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
         return PR_FAILURE;
--- a/pr/src/md/windows/w95thred.c
+++ b/pr/src/md/windows/w95thred.c
@@ -30,31 +30,58 @@ int                           _pr_intsOf
 void
 _PR_MD_EARLY_INIT()
 {
 #ifndef _PR_USE_STATIC_TLS
     _pr_currentThreadIndex = TlsAlloc();
     _pr_lastThreadIndex = TlsAlloc();
     _pr_currentCPUIndex = TlsAlloc();
 #endif
+
+#if defined(_WIN64) && defined(WIN95)
+    _fd_waiting_for_overlapped_done_lock = PR_NewLock();
+#endif
 }
 
 void _PR_MD_CLEANUP_BEFORE_EXIT(void)
 {
     _PR_NT_FreeSids();
 
     _PR_MD_CleanupSockets();
 
     WSACleanup();
 
 #ifndef _PR_USE_STATIC_TLS
     TlsFree(_pr_currentThreadIndex);
     TlsFree(_pr_lastThreadIndex);
     TlsFree(_pr_currentCPUIndex);
 #endif
+
+#if defined(_WIN64) && defined(WIN95)
+    // For each iteration check if TFO overlapped IOs are down.
+    if (_fd_waiting_for_overlapped_done_lock) {
+        PRIntervalTime delay = PR_MillisecondsToInterval(1000);
+        PRFileDescList *cur;
+        do {
+            CheckOverlappedPendingSocketsAreDone();
+
+            PR_Lock(_fd_waiting_for_overlapped_done_lock);
+            cur = _fd_waiting_for_overlapped_done;
+            PR_Unlock(_fd_waiting_for_overlapped_done_lock);
+#if define DO_NOT_WAIT_FOR_CONNECT_OVERLAPPED_OPERATIONS
+            cur = NULL;
+#endif
+            if (cur) {
+                PR_Sleep(delay); // wait another 1s.
+            }
+        } while (cur);
+
+        PR_DestroyLock(_fd_waiting_for_overlapped_done_lock);
+    }
+#endif
 }
 
 PRStatus
 _PR_MD_INIT_THREAD(PRThread *thread)
 {
     if (thread->flags & (_PR_PRIMORDIAL | _PR_ATTACHED)) {
         /*
         ** Warning:
--- a/pr/src/pthreads/ptio.c
+++ b/pr/src/pthreads/ptio.c
@@ -4739,24 +4739,16 @@ PR_IMPLEMENT(PRInt32) PR_FileDesc2Native
     PRInt32 osfd = -1;
     bottom = (NULL == bottom) ?
         NULL : PR_GetIdentitiesLayer(bottom, PR_NSPR_IO_LAYER);
     if (NULL == bottom) PR_SetError(PR_INVALID_ARGUMENT_ERROR, 0);
     else osfd = bottom->secret->md.osfd;
     return osfd;
 }  /* PR_FileDesc2NativeHandle */
 
-/* Expose OVERLAPPED if present. OVERLAPPED is implemented only on WIN95. */
-PR_IMPLEMENT(PRStatus)
-PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(PRFileDesc *fd, void **ol)
-{
-    PR_SetError(PR_NOT_IMPLEMENTED_ERROR, 0);
-    return PR_FAILURE;
-}
-
 PR_IMPLEMENT(void) PR_ChangeFileDescNativeHandle(PRFileDesc *fd,
     PRInt32 handle)
 {
     if (fd) fd->secret->md.osfd = handle;
 }  /*  PR_ChangeFileDescNativeHandle*/
 
 PR_IMPLEMENT(PRStatus) PR_LockFile(PRFileDesc *fd)
 {
--- a/pr/src/pthreads/ptthread.c
+++ b/pr/src/pthreads/ptthread.c
@@ -1059,16 +1059,21 @@ void PR_HPUX10xInit(shl_t handle, int lo
 }
 #endif
 #elif defined(AIX)
 /* Need to use the -binitfini::_PR_Fini linker option. */
 #endif
 
 void _PR_Fini(void)
 {
+    /* We disable the cleanup code on Mac OSX, see bug 1399746.
+     * The .dylib containing NSPR can get unloaded, and _PR_Fini called,
+     * and other code calling NSPR functions can get executed afterwards.
+    */
+#ifndef DARWIN
     void *thred;
     int rv;
 
     if (!_pr_initialized) {
         /* Either NSPR was never successfully initialized or 
          * PR_Cleanup has been called already. */
         if (pt_book.keyCreated)
         {
@@ -1090,16 +1095,17 @@ void _PR_Fini(void)
         rv = pthread_setspecific(pt_book.key, NULL);
         PR_ASSERT(0 == rv);
     }
     rv = pthread_key_delete(pt_book.key);
     PR_ASSERT(0 == rv);
     pt_book.keyCreated = PR_FALSE;
     /* TODO: free other resources used by NSPR */
     /* _pr_initialized = PR_FALSE; */
+#endif
 }  /* _PR_Fini */
 
 PR_IMPLEMENT(PRStatus) PR_Cleanup(void)
 {
     PRThread *me = PR_GetCurrentThread();
     int rv;
     PR_LOG(_pr_thread_lm, PR_LOG_MIN, ("PR_Cleanup: shutting down NSPR"));
     PR_ASSERT(me->state & PT_THREAD_PRIMORD);