Bug 1673313 [Wayland] Don't fail when Shm allocation fails, r=jhorak
☠☠ backed out by cd9aab326ed6 ☠ ☠
authorstransky <stransky@redhat.com>
Tue, 27 Oct 2020 14:21:02 +0000
changeset 554669 dff839f08039beb0d8db6c46daf956b6dfe87fc8
parent 554668 ab66bd46c2d6874336d5fad4faceaa9b9448ff5f
child 554670 371e5ac4d316bce524855138dd678192790bf8b6
push id37898
push userabutkovits@mozilla.com
push dateWed, 28 Oct 2020 09:24:21 +0000
treeherdermozilla-central@83bf4fd3b1fb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhorak
bugs1673313
milestone84.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1673313 [Wayland] Don't fail when Shm allocation fails, r=jhorak - Make WaylandAllocateShmMemory() fallible. - Implement WaylandReAllocateShmMemory() to re-allocate Shm pool. - Remove WaylandShmPool::Resize() and use WaylandShmPool::Create() only. - Implement and use WaylandShmPool::Release(). - Make WindowSurfaceWayland::CreateWaylandBuffer() as fallible. Differential Revision: https://phabricator.services.mozilla.com/D94735
widget/gtk/WindowSurfaceWayland.cpp
widget/gtk/WindowSurfaceWayland.h
--- a/widget/gtk/WindowSurfaceWayland.cpp
+++ b/widget/gtk/WindowSurfaceWayland.cpp
@@ -204,171 +204,215 @@ static void WaylandBufferDelayCommitHand
   }
 }
 
 RefPtr<nsWaylandDisplay> WindowBackBuffer::GetWaylandDisplay() {
   return mWindowSurfaceWayland->GetWaylandDisplay();
 }
 
 static int WaylandAllocateShmMemory(int aSize) {
-  static int counter = 0;
-  nsPrintfCString shmName("/wayland.mozilla.ipc.%d", counter++);
-  int fd = shm_open(shmName.get(), O_CREAT | O_RDWR | O_EXCL, 0600);
-  if (fd >= 0) {
-    shm_unlink(shmName.get());
-  } else {
-    printf_stderr("Unable to SHM memory segment\n");
-    MOZ_CRASH();
+  int fd = -1;
+  do {
+    static int counter = 0;
+    nsPrintfCString shmName("/wayland.mozilla.ipc.%d", counter++);
+    fd = shm_open(shmName.get(), O_CREAT | O_RDWR | O_EXCL, 0600);
+    if (fd >= 0) {
+      // We don't want to use leaked file
+      if (shm_unlink(shmName.get()) != 0) {
+        NS_WARNING("shm_unlink failed");
+        return -1;
+      }
+    }
+  } while (fd < 0 && errno == EEXIST);
+
+  if (fd < 0) {
+    NS_WARNING(nsPrintfCString("shm_open failed: %s", strerror(errno)).get());
+    return -1;
   }
 
   int ret = 0;
 #ifdef HAVE_POSIX_FALLOCATE
   do {
     ret = posix_fallocate(fd, 0, aSize);
   } while (ret == EINTR);
   if (ret != 0) {
+    NS_WARNING(
+        nsPrintfCString("posix_fallocate() fails to allocate shm memory: %s",
+                        strerror(ret))
+            .get());
     close(fd);
-    MOZ_CRASH("posix_fallocate() fails to allocate shm memory");
+    return -1;
   }
 #else
   do {
     ret = ftruncate(fd, aSize);
   } while (ret < 0 && errno == EINTR);
   if (ret < 0) {
+    NS_WARNING(nsPrintfCString("ftruncate() fails to allocate shm memory: %s",
+                               strerror(ret))
+                   .get());
     close(fd);
-    MOZ_CRASH("ftruncate() fails to allocate shm memory");
+    fd = -1;
   }
 #endif
 
   return fd;
 }
 
-WaylandShmPool::WaylandShmPool(RefPtr<nsWaylandDisplay> aWaylandDisplay,
-                               int aSize)
-    : mAllocatedSize(aSize) {
-  mShmPoolFd = WaylandAllocateShmMemory(mAllocatedSize);
-  mImageData = mmap(nullptr, mAllocatedSize, PROT_READ | PROT_WRITE, MAP_SHARED,
-                    mShmPoolFd, 0);
-  MOZ_RELEASE_ASSERT(mImageData != MAP_FAILED,
-                     "Unable to map drawing surface!");
+static bool WaylandReAllocateShmMemory(int aFd, int aSize) {
+  if (ftruncate(aFd, aSize) < 0) {
+    return false;
+  }
+#ifdef HAVE_POSIX_FALLOCATE
+  do {
+    errno = posix_fallocate(aFd, 0, aSize);
+  } while (errno == EINTR);
+  if (errno != 0) {
+    return false;
+  }
+#endif
+  return true;
+}
 
-  mShmPool =
-      wl_shm_create_pool(aWaylandDisplay->GetShm(), mShmPoolFd, mAllocatedSize);
+WaylandShmPool::WaylandShmPool()
+    : mShmPool(nullptr),
+      mShmPoolFd(-1),
+      mAllocatedSize(0),
+      mImageData(MAP_FAILED){};
 
-  // We set our queue to get mShmPool events at compositor thread.
-  wl_proxy_set_queue((struct wl_proxy*)mShmPool,
-                     aWaylandDisplay->GetEventQueue());
+void WaylandShmPool::Release() {
+  if (mImageData != MAP_FAILED) {
+    munmap(mImageData, mAllocatedSize);
+    mImageData = MAP_FAILED;
+  }
+  if (mShmPool) {
+    wl_shm_pool_destroy(mShmPool);
+    mShmPool = 0;
+  }
+  if (mShmPoolFd >= 0) {
+    close(mShmPoolFd);
+    mShmPoolFd = -1;
+  }
 }
 
-bool WaylandShmPool::Resize(int aSize) {
+bool WaylandShmPool::Create(RefPtr<nsWaylandDisplay> aWaylandDisplay,
+                            int aSize) {
   // We do size increase only
-  if (aSize <= mAllocatedSize) return true;
-
-  if (ftruncate(mShmPoolFd, aSize) < 0) return false;
+  if (aSize <= mAllocatedSize) {
+    return true;
+  }
 
-#ifdef HAVE_POSIX_FALLOCATE
-  do {
-    errno = posix_fallocate(mShmPoolFd, 0, aSize);
-  } while (errno == EINTR);
-  if (errno != 0) return false;
-#endif
+  if (mShmPoolFd < 0) {
+    mShmPoolFd = WaylandAllocateShmMemory(aSize);
+    if (mShmPoolFd < 0) {
+      return false;
+    }
+  } else {
+    if (!WaylandReAllocateShmMemory(mShmPoolFd, aSize)) {
+      Release();
+      return false;
+    }
+  }
 
-  wl_shm_pool_resize(mShmPool, aSize);
-
-  munmap(mImageData, mAllocatedSize);
-
+  if (mImageData != MAP_FAILED) {
+    munmap(mImageData, mAllocatedSize);
+  }
   mImageData =
       mmap(nullptr, aSize, PROT_READ | PROT_WRITE, MAP_SHARED, mShmPoolFd, 0);
-  if (mImageData == MAP_FAILED) return false;
+  if (mImageData == MAP_FAILED) {
+    NS_WARNING("Unable to map drawing surface!");
+    Release();
+    return false;
+  }
+
+  if (mShmPool) {
+    wl_shm_pool_resize(mShmPool, aSize);
+  } else {
+    mShmPool = wl_shm_create_pool(aWaylandDisplay->GetShm(), mShmPoolFd, aSize);
+    // We set our queue to get mShmPool events at compositor thread.
+    wl_proxy_set_queue((struct wl_proxy*)mShmPool,
+                       aWaylandDisplay->GetEventQueue());
+  }
 
   mAllocatedSize = aSize;
   return true;
 }
 
 void WaylandShmPool::SetImageDataFromPool(class WaylandShmPool* aSourcePool,
                                           int aImageDataSize) {
   MOZ_ASSERT(mAllocatedSize >= aImageDataSize, "WaylandShmPool overflows!");
   memcpy(mImageData, aSourcePool->GetImageData(), aImageDataSize);
 }
 
-WaylandShmPool::~WaylandShmPool() {
-  munmap(mImageData, mAllocatedSize);
-  wl_shm_pool_destroy(mShmPool);
-  close(mShmPoolFd);
-}
+WaylandShmPool::~WaylandShmPool() { Release(); }
 
 static void buffer_release(void* data, wl_buffer* buffer) {
   auto surface = reinterpret_cast<WindowBackBuffer*>(data);
   surface->Detach(buffer);
 }
 
 static const struct wl_buffer_listener buffer_listener = {buffer_release};
 
-void WindowBackBufferShm::Create(int aWidth, int aHeight) {
+bool WindowBackBufferShm::Create(int aWidth, int aHeight) {
   MOZ_ASSERT(!IsAttached(), "We can't create attached buffers.");
-  MOZ_ASSERT(!mWLBuffer, "there is wl_buffer already!");
 
-  int newBufferSize = aWidth * aHeight * BUFFER_BPP;
-  if (!mShmPool.Resize(newBufferSize)) {
-    mWLBuffer = nullptr;
-    return;
+  ReleaseShmSurface();
+
+  int size = aWidth * aHeight * BUFFER_BPP;
+  if (!mShmPool.Create(GetWaylandDisplay(), size)) {
+    return false;
   }
 
   mWLBuffer =
       wl_shm_pool_create_buffer(mShmPool.GetShmPool(), 0, aWidth, aHeight,
                                 aWidth * BUFFER_BPP, WL_SHM_FORMAT_ARGB8888);
   wl_proxy_set_queue((struct wl_proxy*)mWLBuffer,
                      GetWaylandDisplay()->GetEventQueue());
   wl_buffer_add_listener(mWLBuffer, &buffer_listener, this);
 
   mWidth = aWidth;
   mHeight = aHeight;
 
   LOGWAYLAND(("WindowBackBufferShm::Create [%p] wl_buffer %p ID %d\n",
               (void*)this, (void*)mWLBuffer,
               mWLBuffer ? wl_proxy_get_id((struct wl_proxy*)mWLBuffer) : -1));
+  return true;
 }
 
 void WindowBackBufferShm::ReleaseShmSurface() {
   LOGWAYLAND(("WindowBackBufferShm::Release [%p]\n", (void*)this));
-
-  wl_buffer_destroy(mWLBuffer);
+  if (mWLBuffer) {
+    wl_buffer_destroy(mWLBuffer);
+    mWLBuffer = nullptr;
+  }
   mWidth = mHeight = 0;
-  mWLBuffer = nullptr;
 }
 
 void WindowBackBufferShm::Clear() {
   memset(mShmPool.GetImageData(), 0, mHeight * mWidth * BUFFER_BPP);
 }
 
 WindowBackBufferShm::WindowBackBufferShm(
-    WindowSurfaceWayland* aWindowSurfaceWayland, int aWidth, int aHeight)
+    WindowSurfaceWayland* aWindowSurfaceWayland)
     : WindowBackBuffer(aWindowSurfaceWayland),
-      mShmPool(aWindowSurfaceWayland->GetWaylandDisplay(),
-               aWidth * aHeight * BUFFER_BPP),
+      mShmPool(),
       mWLBuffer(nullptr),
-      mWidth(aWidth),
-      mHeight(aHeight),
-      mAttached(false) {
-  Create(aWidth, aHeight);
-}
+      mWidth(0),
+      mHeight(0),
+      mAttached(false) {}
 
 WindowBackBufferShm::~WindowBackBufferShm() { ReleaseShmSurface(); }
 
 bool WindowBackBufferShm::Resize(int aWidth, int aHeight) {
   if (aWidth == mWidth && aHeight == mHeight) {
     return true;
   }
-
   LOGWAYLAND(("WindowBackBufferShm::Resize [%p] %d %d\n", (void*)this, aWidth,
               aHeight));
-
-  ReleaseShmSurface();
   Create(aWidth, aHeight);
-
   return (mWLBuffer != nullptr);
 }
 
 void WindowBackBuffer::Attach(wl_surface* aSurface) {
   LOGWAYLAND(
       ("WindowBackBuffer::Attach [%p] wl_surface %p ID %d wl_buffer %p ID %d\n",
        (void*)this, (void*)aSurface,
        aSurface ? wl_proxy_get_id((struct wl_proxy*)aSurface) : -1,
@@ -483,21 +527,23 @@ WindowBackBuffer* WindowSurfaceWayland::
     }
   }
 
   // There isn't any free slot for additional buffer.
   if (availableBuffer == BACK_BUFFER_NUM) {
     return nullptr;
   }
 
-  WindowBackBuffer* buffer = new WindowBackBufferShm(this, aWidth, aHeight);
-  if (buffer) {
-    mShmBackupBuffer[availableBuffer] = buffer;
+  WindowBackBuffer* buffer = new WindowBackBufferShm(this);
+  if (!buffer->Create(aWidth, aHeight)) {
+    delete buffer;
+    return nullptr;
   }
 
+  mShmBackupBuffer[availableBuffer] = buffer;
   return buffer;
 }
 
 WindowBackBuffer* WindowSurfaceWayland::WaylandBufferFindAvailable(
     int aWidth, int aHeight) {
   int availableBuffer;
   // Try to find a buffer which matches the size
   for (availableBuffer = 0; availableBuffer < BACK_BUFFER_NUM;
--- a/widget/gtk/WindowSurfaceWayland.h
+++ b/widget/gtk/WindowSurfaceWayland.h
@@ -20,24 +20,24 @@
 namespace mozilla {
 namespace widget {
 
 class WindowSurfaceWayland;
 
 // Allocates and owns shared memory for Wayland drawing surface
 class WaylandShmPool {
  public:
-  WaylandShmPool(RefPtr<nsWaylandDisplay> aDisplay, int aSize);
-  ~WaylandShmPool();
-
-  bool Resize(int aSize);
+  bool Create(RefPtr<nsWaylandDisplay> aWaylandDisplay, int aSize);
+  void Release();
   wl_shm_pool* GetShmPool() { return mShmPool; };
   void* GetImageData() { return mImageData; };
   void SetImageDataFromPool(class WaylandShmPool* aSourcePool,
                             int aImageDataSize);
+  WaylandShmPool();
+  ~WaylandShmPool();
 
  private:
   wl_shm_pool* mShmPool;
   int mShmPoolFd;
   int mAllocatedSize;
   void* mImageData;
 };
 
@@ -48,16 +48,17 @@ class WindowBackBuffer {
   virtual void Unlock() = 0;
   virtual bool IsLocked() = 0;
 
   void Attach(wl_surface* aSurface);
   virtual void Detach(wl_buffer* aBuffer) = 0;
   virtual bool IsAttached() = 0;
 
   virtual void Clear() = 0;
+  virtual bool Create(int aWidth, int aHeight) = 0;
   virtual bool Resize(int aWidth, int aHeight) = 0;
 
   virtual int GetWidth() = 0;
   virtual int GetHeight() = 0;
   virtual wl_buffer* GetWlBuffer() = 0;
   virtual void SetAttached() = 0;
 
   virtual bool SetImageDataFromBuffer(
@@ -82,39 +83,38 @@ class WindowBackBuffer {
   WindowSurfaceWayland* mWindowSurfaceWayland;
 
  private:
   static gfx::SurfaceFormat mFormat;
 };
 
 class WindowBackBufferShm : public WindowBackBuffer {
  public:
-  WindowBackBufferShm(WindowSurfaceWayland* aWindowSurfaceWayland, int aWidth,
-                      int aHeight);
+  WindowBackBufferShm(WindowSurfaceWayland* aWindowSurfaceWayland);
   ~WindowBackBufferShm();
 
   already_AddRefed<gfx::DrawTarget> Lock();
   bool IsLocked() { return mIsLocked; };
   void Unlock() { mIsLocked = false; };
 
   void Detach(wl_buffer* aBuffer);
   bool IsAttached() { return mAttached; }
   void SetAttached() { mAttached = true; };
 
   void Clear();
+  bool Create(int aWidth, int aHeight);
   bool Resize(int aWidth, int aHeight);
   bool SetImageDataFromBuffer(class WindowBackBuffer* aSourceBuffer);
 
   int GetWidth() { return mWidth; };
   int GetHeight() { return mHeight; };
 
   wl_buffer* GetWlBuffer() { return mWLBuffer; };
 
  private:
-  void Create(int aWidth, int aHeight);
   void ReleaseShmSurface();
 
   // WaylandShmPool provides actual shared memory we draw into
   WaylandShmPool mShmPool;
 
   // wl_buffer is a wayland object that encapsulates the shared memory
   // and passes it to wayland compositor by wl_surface object.
   wl_buffer* mWLBuffer;