Bug 1556993 - Ignore invalid handles in Mozilla{,Un}RegisterDebugHandle - r=erahm
authorGerald Squelart <gsquelart@mozilla.com>
Wed, 12 Jun 2019 23:23:20 +0000
changeset 478572 a01cf6b0fc83e480637774c4636a3d40169176fd
parent 478563 4a63f0a3a1f26e2a377ffbd477ba050e16577445
child 478573 547109ab8d7bb037fa42996e1837f7d231b5d615
push id113428
push useropoprus@mozilla.com
push dateThu, 13 Jun 2019 10:07:38 +0000
treeherdermozilla-inbound@e93ecdda786c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerserahm
bugs1556993
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1556993 - Ignore invalid handles in Mozilla{,Un}RegisterDebugHandle - r=erahm In particular, on Windows stdin/out/err may not be associated with a stream (e.g., app without console window), in which case _get_osfhandle() returns -2. Differential Revision: https://phabricator.services.mozilla.com/D34513
xpcom/build/PoisonIOInterposerBase.cpp
--- a/xpcom/build/PoisonIOInterposerBase.cpp
+++ b/xpcom/build/PoisonIOInterposerBase.cpp
@@ -1,14 +1,15 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "mozilla/Maybe.h"
 #include "mozilla/Mutex.h"
 #include "mozilla/Scoped.h"
 #include "mozilla/UniquePtr.h"
 
 #include <algorithm>
 
 #include "PoisonIOInterposer.h"
 
@@ -16,19 +17,28 @@
 
 #ifdef MOZ_REPLACE_MALLOC
 #  include "replace_malloc_bridge.h"
 #endif
 
 // Auxiliary method to convert file descriptors to ids
 #if defined(XP_WIN)
 #  include <io.h>
-inline intptr_t FileDescriptorToHandle(int aFd) { return _get_osfhandle(aFd); }
+inline mozilla::Maybe<intptr_t> FileDescriptorToHandle(int aFd) {
+  intptr_t handle = _get_osfhandle(aFd);
+  if ((handle == -1) || (handle == -2)) {
+    // -1: Invalid handle. -2: stdin/out/err not associated with a stream.
+    return mozilla::Nothing();
+  }
+  return mozilla::Some(handle);
+}
 #else
-inline intptr_t FileDescriptorToHandle(int aFd) { return aFd; }
+inline mozilla::Maybe<intptr_t> FileDescriptorToHandle(int aFd) {
+  return mozilla::Some<intptr_t>(aFd);
+}
 #endif /* if not XP_WIN */
 
 using namespace mozilla;
 
 namespace {
 
 struct DebugFilesAutoLockTraits {
   typedef PRLock* type;
@@ -203,17 +213,21 @@ extern "C" {
 void MozillaRegisterDebugHandle(intptr_t aHandle) {
   DebugFilesAutoLock lockedScope;
   FdList& DebugFileIDs = getDebugFileIDs();
   MOZ_ASSERT(!DebugFileIDs.Contains(aHandle));
   DebugFileIDs.Add(aHandle);
 }
 
 void MozillaRegisterDebugFD(int aFd) {
-  MozillaRegisterDebugHandle(FileDescriptorToHandle(aFd));
+  Maybe<intptr_t> handle = FileDescriptorToHandle(aFd);
+  if (!handle.isSome()) {
+    return;
+  }
+  MozillaRegisterDebugHandle(handle.value());
 }
 
 void MozillaRegisterDebugFILE(FILE* aFile) {
   int fd = fileno(aFile);
   if (fd == 1 || fd == 2) {
     return;
   }
   MozillaRegisterDebugFD(fd);
@@ -222,17 +236,21 @@ void MozillaRegisterDebugFILE(FILE* aFil
 void MozillaUnRegisterDebugHandle(intptr_t aHandle) {
   DebugFilesAutoLock lockedScope;
   FdList& DebugFileIDs = getDebugFileIDs();
   MOZ_ASSERT(DebugFileIDs.Contains(aHandle));
   DebugFileIDs.Remove(aHandle);
 }
 
 void MozillaUnRegisterDebugFD(int aFd) {
-  MozillaUnRegisterDebugHandle(FileDescriptorToHandle(aFd));
+  Maybe<intptr_t> handle = FileDescriptorToHandle(aFd);
+  if (!handle.isSome()) {
+    return;
+  }
+  MozillaUnRegisterDebugHandle(handle.value());
 }
 
 void MozillaUnRegisterDebugFILE(FILE* aFile) {
   int fd = fileno(aFile);
   if (fd == 1 || fd == 2) {
     return;
   }
   fflush(aFile);