Bug 804606 - Stop Flash from crashing in CoreGraphics mode on accessing "our" CGContextRef outside of the call we use to pass it. r=bgirard a=bbajaj
authorSteven Michaud <smichaud@pobox.com>
Tue, 18 Dec 2012 15:03:40 -0600
changeset 118120 3fbda0a1a6e9e04928fd1c47fd1ed553841167f4
parent 118119 cc071482752bf1c8347eef2146011129787cb934
child 118121 f2fcf38b2b5654f3db9782d76843ec9aeceddb25
push id239
push userakeybl@mozilla.com
push dateThu, 03 Jan 2013 21:54:43 +0000
treeherdermozilla-release@3a7b66445659 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbgirard, bbajaj
bugs804606
milestone18.0
Bug 804606 - Stop Flash from crashing in CoreGraphics mode on accessing "our" CGContextRef outside of the call we use to pass it. r=bgirard a=bbajaj
dom/plugins/ipc/PluginInstanceChild.cpp
dom/plugins/ipc/PluginModuleChild.cpp
dom/plugins/ipc/PluginModuleChild.h
dom/plugins/ipc/PluginUtilsOSX.h
dom/plugins/ipc/PluginUtilsOSX.mm
widget/cocoa/nsCocoaFeatures.h
widget/cocoa/nsCocoaFeatures.mm
--- a/dom/plugins/ipc/PluginInstanceChild.cpp
+++ b/dom/plugins/ipc/PluginInstanceChild.cpp
@@ -3041,17 +3041,20 @@ PluginInstanceChild::EnsureCurrentBuffer
 
     return true;
 #else // XP_MACOSX
 
     if (!mDoubleBufferCARenderer.HasCALayer()) {
         void *caLayer = nullptr;
         if (mDrawingModel == NPDrawingModelCoreGraphics) {
             if (!mCGLayer) {
-                caLayer = mozilla::plugins::PluginUtilsOSX::GetCGLayer(CallCGDraw, this);
+                bool avoidCGCrashes = !nsCocoaFeatures::OnMountainLionOrLater() &&
+                  (GetQuirks() & PluginModuleChild::QUIRK_FLASH_AVOID_CGMODE_CRASHES);
+                caLayer = mozilla::plugins::PluginUtilsOSX::GetCGLayer(CallCGDraw, this,
+                                                                       avoidCGCrashes);
 
                 if (!caLayer) {
                     PLUGIN_LOG_DEBUG(("GetCGLayer failed."));
                     return false;
                 }
             }
             mCGLayer = caLayer;
         } else {
--- a/dom/plugins/ipc/PluginModuleChild.cpp
+++ b/dom/plugins/ipc/PluginModuleChild.cpp
@@ -164,25 +164,33 @@ PluginModuleChild::Init(const std::strin
     bool exists;
     localFile->Exists(&exists);
     NS_ASSERTION(exists, "plugin file ain't there");
 
     nsPluginFile pluginFile(localFile);
 
     // Maemo flash can render with any provided rectangle and so does not
     // require this quirk.
-#if defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)
+#if (defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)) || defined(OS_MACOSX)
     nsPluginInfo info = nsPluginInfo();
     if (NS_FAILED(pluginFile.GetPluginInfo(info, &mLibrary)))
         return false;
 
+#if defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)
     NS_NAMED_LITERAL_CSTRING(flash10Head, "Shockwave Flash 10.");
     if (StringBeginsWith(nsDependentCString(info.fDescription), flash10Head)) {
         AddQuirk(QUIRK_FLASH_EXPOSE_COORD_TRANSLATION);
     }
+#else // defined(OS_MACOSX)
+    mozilla::plugins::PluginUtilsOSX::SetProcessName(info.fName);
+    NS_NAMED_LITERAL_CSTRING(flashHead, "Shockwave Flash");
+    if (StringBeginsWith(nsDependentCString(info.fDescription), flashHead)) {
+        AddQuirk(QUIRK_FLASH_AVOID_CGMODE_CRASHES);
+    }
+#endif
 
     if (!mLibrary)
 #endif
     {
         nsresult rv = pluginFile.LoadPlugin(&mLibrary);
         if (NS_FAILED(rv))
             return false;
     }
@@ -219,23 +227,16 @@ PluginModuleChild::Init(const std::strin
         (NP_PLUGININIT)PR_FindFunctionSymbol(mLibrary, "NP_Initialize");
     NS_ENSURE_TRUE(mInitializeFunc, false);
 #else
 
 #  error Please copy the initialization code from nsNPAPIPlugin.cpp
 
 #endif
 
-#ifdef XP_MACOSX
-    nsPluginInfo info = nsPluginInfo();
-    if (pluginFile.GetPluginInfo(info, &mLibrary) == NS_OK) {
-        mozilla::plugins::PluginUtilsOSX::SetProcessName(info.fName);
-    }
-#endif
-
     return true;
 }
 
 #if defined(MOZ_WIDGET_GTK)
 typedef void (*GObjectDisposeFn)(GObject*);
 typedef gboolean (*GtkWidgetScrollEventFn)(GtkWidget*, GdkEventScroll*);
 typedef void (*GtkPlugEmbeddedFn)(GtkPlug*);
 
--- a/dom/plugins/ipc/PluginModuleChild.h
+++ b/dom/plugins/ipc/PluginModuleChild.h
@@ -282,16 +282,21 @@ public:
         QUIRK_QUICKTIME_AVOID_SETWINDOW                 = 1 << 7,
         // Win: Check to make sure the parent window has focus before calling
         // set focus on the child. Addresses a full screen dialog prompt
         // problem in Silverlight.
         QUIRK_SILVERLIGHT_FOCUS_CHECK_PARENT            = 1 << 8,
         // Mac: Allow the plugin to use offline renderer mode.
         // Use this only if the plugin is certified the support the offline renderer.
         QUIRK_ALLOW_OFFLINE_RENDERER                    = 1 << 9,
+        // Mac: Work around a Flash bug that can cause plugin process crashes
+        // in CoreGraphics mode:  The Flash plugin sometimes accesses the
+        // CGContextRef we pass to it in NPP_HandleEvent(NPCocoaEventDrawRect)
+        // outside of that call.  See bug 804606.
+        QUIRK_FLASH_AVOID_CGMODE_CRASHES                = 1 << 10,
     };
 
     int GetQuirks() { return mQuirks; }
 
 private:
     void AddQuirk(PluginQuirks quirk) {
       if (mQuirks == QUIRKS_NOT_INITIALIZED)
         mQuirks = 0;
--- a/dom/plugins/ipc/PluginUtilsOSX.h
+++ b/dom/plugins/ipc/PluginUtilsOSX.h
@@ -20,17 +20,17 @@ typedef void (*RemoteProcessEvents) (voi
 
 NPError ShowCocoaContextMenu(void* aMenu, int aX, int aY, void* pluginModule, RemoteProcessEvents remoteEvent);
 
 void InvokeNativeEventLoop();
 
 // Need to call back and send a cocoa draw event to the plugin.
 typedef void (*DrawPluginFunc) (CGContextRef, void*, nsIntRect aUpdateRect);
 
-void* GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance);
+void* GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance, bool aAvoidCGCrashes);
 void ReleaseCGLayer(void* cgLayer);
 void Repaint(void* cgLayer, nsIntRect aRect);
 
 bool SetProcessName(const char* aProcessName);
 
 /*
  * Provides a wrapper around nsCARenderer to manage double buffering
  * without having to unbind nsCARenderer on every surface swaps.
--- a/dom/plugins/ipc/PluginUtilsOSX.mm
+++ b/dom/plugins/ipc/PluginUtilsOSX.mm
@@ -1,14 +1,15 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 // vim:set ts=2 sts=2 sw=2 et cin:
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include <dlfcn.h>
 #import <AppKit/AppKit.h>
 #import <QuartzCore/QuartzCore.h>
 #include "PluginUtilsOSX.h"
 
 // Remove definitions for try/catch interfering with ObjCException macros.
 #include "nsObjCExceptions.h"
 #include "nsCocoaUtils.h"
 
@@ -20,68 +21,144 @@
 @end
 
 using namespace mozilla::plugins::PluginUtilsOSX;
 
 @interface CGBridgeLayer : CALayer {
   DrawPluginFunc mDrawFunc;
   void* mPluginInstance;
   nsIntRect mUpdateRect;
+  BOOL mAvoidCGCrashes;
+  CGContextRef mLastCGContext;
 }
-- (void) setDrawFunc: (DrawPluginFunc)aFunc pluginInstance:(void*) aPluginInstance;
-- (void) updateRect: (nsIntRect)aRect;
+- (void)setDrawFunc:(DrawPluginFunc)aFunc
+     pluginInstance:(void*)aPluginInstance
+     avoidCGCrashes:(BOOL)aAvoidCGCrashes;
+- (void)updateRect:(nsIntRect)aRect;
+- (void)protectLastCGContext;
 
 @end
 
+// CGBitmapContextSetData() is an undocumented function present (with
+// the same signature) since at least OS X 10.5.  As the name suggests,
+// it's used to replace the "data" in a bitmap context that was
+// originally specified in a call to CGBitmapContextCreate() or
+// CGBitmapContextCreateWithData().
+typedef void (*CGBitmapContextSetDataFunc) (CGContextRef c,
+                                            size_t x,
+                                            size_t y,
+                                            size_t width,
+                                            size_t height,
+                                            void* data,
+                                            size_t bitsPerComponent,
+                                            size_t bitsPerPixel,
+                                            size_t bytesPerRow);
+CGBitmapContextSetDataFunc CGBitmapContextSetDataPtr = NULL;
+
 @implementation CGBridgeLayer
-- (void) updateRect: (nsIntRect)aRect
+- (void) updateRect:(nsIntRect)aRect
 {
    mUpdateRect.UnionRect(mUpdateRect, aRect);
 }
 
-- (void) setDrawFunc: (DrawPluginFunc)aFunc pluginInstance:(void*) aPluginInstance
+- (void) setDrawFunc:(DrawPluginFunc)aFunc
+      pluginInstance:(void*)aPluginInstance
+      avoidCGCrashes:(BOOL)aAvoidCGCrashes
 {
   mDrawFunc = aFunc;
   mPluginInstance = aPluginInstance;
+  mAvoidCGCrashes = aAvoidCGCrashes;
+  mLastCGContext = nil;
+}
+
+// The Flash plugin, in very unusual circumstances, can (in CoreGraphics
+// mode) try to access the CGContextRef from -[CGBridgeLayer drawInContext:]
+// outside of any call to NPP_HandleEvent(NPCocoaEventDrawRect).  This usually
+// crashes the plugin process (probably because it tries to access deleted
+// memory).  We stop these crashes from happening by holding a reference to
+// the CGContextRef, and also by ensuring that it's data won't get deleted.
+// The CGContextRef won't "work" in this form.  But this won't cause trouble
+// for plugins that do things correctly (that don't access this CGContextRef
+// outside of the call to NPP_HandleEvent() that passes it to the plugin).
+// The OS may reuse this CGContextRef (it may get passed to other calls to
+// -[CGBridgeLayer drawInContext:]).  But before each call the OS calls
+// CGBitmapContextSetData() to replace its data, which undoes the changes
+// we make here.  See bug 804606.
+- (void)protectLastCGContext
+{
+  if (!mAvoidCGCrashes || !mLastCGContext) {
+    return;
+  }
+
+  static char ensuredData[128] = {0};
+
+  if (!CGBitmapContextSetDataPtr) {
+    CGBitmapContextSetDataPtr = (CGBitmapContextSetDataFunc)
+      dlsym(RTLD_DEFAULT, "CGBitmapContextSetData");
+  }
+
+  if (CGBitmapContextSetDataPtr && (GetContextType(mLastCGContext) == CG_CONTEXT_TYPE_BITMAP)) {
+    CGBitmapContextSetDataPtr(mLastCGContext, 0, 0, 1, 1, ensuredData, 8, 32, 64);
+  }
 }
 
 - (void)drawInContext:(CGContextRef)aCGContext
-
 {
   ::CGContextSaveGState(aCGContext); 
   ::CGContextTranslateCTM(aCGContext, 0, self.bounds.size.height);
   ::CGContextScaleCTM(aCGContext, (CGFloat) 1, (CGFloat) -1);
 
   mDrawFunc(aCGContext, mPluginInstance, mUpdateRect);
 
-  ::CGContextRestoreGState(aCGContext); 
+  ::CGContextRestoreGState(aCGContext);
+
+  if (mAvoidCGCrashes) {
+    if (mLastCGContext) {
+      ::CGContextRelease(mLastCGContext);
+    }
+    mLastCGContext = aCGContext;
+    ::CGContextRetain(mLastCGContext);
+  }
 
   mUpdateRect.SetEmpty();
 }
 
+- (void)dealloc
+{
+  if (mLastCGContext) {
+    ::CGContextRelease(mLastCGContext);
+  }
+  [super dealloc];
+}
+
 @end
 
-void* mozilla::plugins::PluginUtilsOSX::GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance) {
+void* mozilla::plugins::PluginUtilsOSX::GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance,
+                                                   bool aAvoidCGCrashes)
+{
   CGBridgeLayer *bridgeLayer = [[CGBridgeLayer alloc] init ];
-  [bridgeLayer setDrawFunc:aFunc pluginInstance:aPluginInstance];
+  [bridgeLayer setDrawFunc:aFunc
+            pluginInstance:aPluginInstance
+            avoidCGCrashes:aAvoidCGCrashes];
   return bridgeLayer;
 }
 
 void mozilla::plugins::PluginUtilsOSX::ReleaseCGLayer(void *cgLayer) {
   CGBridgeLayer *bridgeLayer = (CGBridgeLayer*)cgLayer;
   [bridgeLayer release];
 }
 
 void mozilla::plugins::PluginUtilsOSX::Repaint(void *caLayer, nsIntRect aRect) {
   CGBridgeLayer *bridgeLayer = (CGBridgeLayer*)caLayer;
   [CATransaction begin];
   [bridgeLayer updateRect:aRect];
   [bridgeLayer setNeedsDisplay];
   [bridgeLayer displayIfNeeded];
   [CATransaction commit];
+  [bridgeLayer protectLastCGContext];
 }
 
 @interface EventProcessor : NSObject {
   RemoteProcessEvents   aRemoteEvents;
   void                 *aPluginModule;
 }
 - (void)setRemoteEvents:(RemoteProcessEvents) remoteEvents pluginModule:(void*) pluginModule;
 - (void)onTick;
--- a/widget/cocoa/nsCocoaFeatures.h
+++ b/widget/cocoa/nsCocoaFeatures.h
@@ -8,14 +8,15 @@
 
 #include "prtypes.h"
 
 class nsCocoaFeatures {
 public:
   static int32_t OSXVersion();
   static bool OnSnowLeopardOrLater();
   static bool OnLionOrLater();
+  static bool OnMountainLionOrLater();
   static bool SupportCoreAnimationPlugins();
 
 private:
   static int32_t mOSXVersion;
 };
 #endif // nsCocoaFeatures_h_
--- a/widget/cocoa/nsCocoaFeatures.mm
+++ b/widget/cocoa/nsCocoaFeatures.mm
@@ -3,16 +3,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #define MAC_OS_X_VERSION_MASK     0x0000FFFF // Not supported
 #define MAC_OS_X_VERSION_10_4_HEX 0x00001040 // Not supported
 #define MAC_OS_X_VERSION_10_5_HEX 0x00001050
 #define MAC_OS_X_VERSION_10_6_HEX 0x00001060
 #define MAC_OS_X_VERSION_10_7_HEX 0x00001070
+#define MAC_OS_X_VERSION_10_8_HEX 0x00001080
 
 // This API will not work for OS X 10.10, see Gestalt.h.
 
 #include "nsCocoaFeatures.h"
 #include "nsDebug.h"
 #include "nsObjCExceptions.h"
 
 #import <Cocoa/Cocoa.h>
@@ -55,8 +56,14 @@ nsCocoaFeatures::OnSnowLeopardOrLater()
 }
 
 /* static */ bool
 nsCocoaFeatures::OnLionOrLater()
 {
     return (OSXVersion() >= MAC_OS_X_VERSION_10_7_HEX);
 }
 
+/* static */ bool
+nsCocoaFeatures::OnMountainLionOrLater()
+{
+    return (OSXVersion() >= MAC_OS_X_VERSION_10_8_HEX);
+}
+