Bug 804606 - Stop Flash from crashing in CoreGraphics mode on accessing "our" CGContextRef outside of the call we use to pass it. r=bgirard

dom/plugins/ipc/PluginInstanceChild.cpp

@@ -3047,7 +3047,10 @@ PluginInstanceChild::EnsureCurrentBuffer(void)
         void *caLayer = nullptr;
         if (mDrawingModel == NPDrawingModelCoreGraphics) {
             if (!mCGLayer) {
-                caLayer = mozilla::plugins::PluginUtilsOSX::GetCGLayer(CallCGDraw, this);
+                bool avoidCGCrashes = !nsCocoaFeatures::OnMountainLionOrLater() &&
+                  (GetQuirks() & PluginModuleChild::QUIRK_FLASH_AVOID_CGMODE_CRASHES);
+                caLayer = mozilla::plugins::PluginUtilsOSX::GetCGLayer(CallCGDraw, this,
+                                                                       avoidCGCrashes);
 
                 if (!caLayer) {
                     PLUGIN_LOG_DEBUG(("GetCGLayer failed."));

dom/plugins/ipc/PluginModuleChild.cpp

@@ -169,15 +169,23 @@ PluginModuleChild::Init(const std::string& aPluginFilename,
 
     // Maemo flash can render with any provided rectangle and so does not
     // require this quirk.
-#if defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)
+#if (defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)) || defined(OS_MACOSX)
     nsPluginInfo info = nsPluginInfo();
     if (NS_FAILED(pluginFile.GetPluginInfo(info, &mLibrary)))
         return false;
 
+#if defined(MOZ_X11) && !defined(MOZ_PLATFORM_MAEMO)
     NS_NAMED_LITERAL_CSTRING(flash10Head, "Shockwave Flash 10.");
     if (StringBeginsWith(nsDependentCString(info.fDescription), flash10Head)) {
         AddQuirk(QUIRK_FLASH_EXPOSE_COORD_TRANSLATION);
     }
+#else // defined(OS_MACOSX)
+    mozilla::plugins::PluginUtilsOSX::SetProcessName(info.fName);
+    NS_NAMED_LITERAL_CSTRING(flashHead, "Shockwave Flash");
+    if (StringBeginsWith(nsDependentCString(info.fDescription), flashHead)) {
+        AddQuirk(QUIRK_FLASH_AVOID_CGMODE_CRASHES);
+    }
+#endif
 
     if (!mLibrary)
 #endif
@@ -222,13 +230,6 @@ PluginModuleChild::Init(const std::string& aPluginFilename,
 
 #  error Please copy the initialization code from nsNPAPIPlugin.cpp
 
-#endif
-
-#ifdef XP_MACOSX
-    nsPluginInfo info = nsPluginInfo();
-    if (pluginFile.GetPluginInfo(info, &mLibrary) == NS_OK) {
-        mozilla::plugins::PluginUtilsOSX::SetProcessName(info.fName);
-    }
 #endif
 
     return true;

dom/plugins/ipc/PluginModuleChild.h

@@ -287,6 +287,11 @@ public:
         // Mac: Allow the plugin to use offline renderer mode.
         // Use this only if the plugin is certified the support the offline renderer.
         QUIRK_ALLOW_OFFLINE_RENDERER                    = 1 << 9,
+        // Mac: Work around a Flash bug that can cause plugin process crashes
+        // in CoreGraphics mode:  The Flash plugin sometimes accesses the
+        // CGContextRef we pass to it in NPP_HandleEvent(NPCocoaEventDrawRect)
+        // outside of that call.  See bug 804606.
+        QUIRK_FLASH_AVOID_CGMODE_CRASHES                = 1 << 10,
     };
 
     int GetQuirks() { return mQuirks; }

dom/plugins/ipc/PluginUtilsOSX.h

@@ -25,7 +25,7 @@ void InvokeNativeEventLoop();
 // Need to call back and send a cocoa draw event to the plugin.
 typedef void (*DrawPluginFunc) (CGContextRef, void*, nsIntRect aUpdateRect);
 
-void* GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance);
+void* GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance, bool aAvoidCGCrashes);
 void ReleaseCGLayer(void* cgLayer);
 void Repaint(void* cgLayer, nsIntRect aRect);
 

dom/plugins/ipc/PluginUtilsOSX.mm

@@ -4,6 +4,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include <dlfcn.h>
 #import <AppKit/AppKit.h>
 #import <QuartzCore/QuartzCore.h>
 #include "PluginUtilsOSX.h"
@@ -25,26 +26,81 @@ using namespace mozilla::plugins::PluginUtilsOSX;
   DrawPluginFunc mDrawFunc;
   void* mPluginInstance;
   nsIntRect mUpdateRect;
+  BOOL mAvoidCGCrashes;
+  CGContextRef mLastCGContext;
 }
-- (void) setDrawFunc: (DrawPluginFunc)aFunc pluginInstance:(void*) aPluginInstance;
-- (void) updateRect: (nsIntRect)aRect;
+- (void)setDrawFunc:(DrawPluginFunc)aFunc
+     pluginInstance:(void*)aPluginInstance
+     avoidCGCrashes:(BOOL)aAvoidCGCrashes;
+- (void)updateRect:(nsIntRect)aRect;
+- (void)protectLastCGContext;
 
 @end
 
+// CGBitmapContextSetData() is an undocumented function present (with
+// the same signature) since at least OS X 10.5.  As the name suggests,
+// it's used to replace the "data" in a bitmap context that was
+// originally specified in a call to CGBitmapContextCreate() or
+// CGBitmapContextCreateWithData().
+typedef void (*CGBitmapContextSetDataFunc) (CGContextRef c,
+                                            size_t x,
+                                            size_t y,
+                                            size_t width,
+                                            size_t height,
+                                            void* data,
+                                            size_t bitsPerComponent,
+                                            size_t bitsPerPixel,
+                                            size_t bytesPerRow);
+CGBitmapContextSetDataFunc CGBitmapContextSetDataPtr = NULL;
+
 @implementation CGBridgeLayer
-- (void) updateRect: (nsIntRect)aRect
+- (void) updateRect:(nsIntRect)aRect
 {
    mUpdateRect.UnionRect(mUpdateRect, aRect);
 }
 
-- (void) setDrawFunc: (DrawPluginFunc)aFunc pluginInstance:(void*) aPluginInstance
+- (void) setDrawFunc:(DrawPluginFunc)aFunc
+      pluginInstance:(void*)aPluginInstance
+      avoidCGCrashes:(BOOL)aAvoidCGCrashes
 {
   mDrawFunc = aFunc;
   mPluginInstance = aPluginInstance;
+  mAvoidCGCrashes = aAvoidCGCrashes;
+  mLastCGContext = nil;
+}
+
+// The Flash plugin, in very unusual circumstances, can (in CoreGraphics
+// mode) try to access the CGContextRef from -[CGBridgeLayer drawInContext:]
+// outside of any call to NPP_HandleEvent(NPCocoaEventDrawRect).  This usually
+// crashes the plugin process (probably because it tries to access deleted
+// memory).  We stop these crashes from happening by holding a reference to
+// the CGContextRef, and also by ensuring that it's data won't get deleted.
+// The CGContextRef won't "work" in this form.  But this won't cause trouble
+// for plugins that do things correctly (that don't access this CGContextRef
+// outside of the call to NPP_HandleEvent() that passes it to the plugin).
+// The OS may reuse this CGContextRef (it may get passed to other calls to
+// -[CGBridgeLayer drawInContext:]).  But before each call the OS calls
+// CGBitmapContextSetData() to replace its data, which undoes the changes
+// we make here.  See bug 804606.
+- (void)protectLastCGContext
+{
+  if (!mAvoidCGCrashes || !mLastCGContext) {
+    return;
+  }
+
+  static char ensuredData[128] = {0};
+
+  if (!CGBitmapContextSetDataPtr) {
+    CGBitmapContextSetDataPtr = (CGBitmapContextSetDataFunc)
+      dlsym(RTLD_DEFAULT, "CGBitmapContextSetData");
+  }
+
+  if (CGBitmapContextSetDataPtr && (GetContextType(mLastCGContext) == CG_CONTEXT_TYPE_BITMAP)) {
+    CGBitmapContextSetDataPtr(mLastCGContext, 0, 0, 1, 1, ensuredData, 8, 32, 64);
+  }
 }
 
 - (void)drawInContext:(CGContextRef)aCGContext
-
 {
   ::CGContextSaveGState(aCGContext); 
   ::CGContextTranslateCTM(aCGContext, 0, self.bounds.size.height);
@@ -52,16 +108,36 @@ using namespace mozilla::plugins::PluginUtilsOSX;
 
   mDrawFunc(aCGContext, mPluginInstance, mUpdateRect);
 
-  ::CGContextRestoreGState(aCGContext); 
+  ::CGContextRestoreGState(aCGContext);
+
+  if (mAvoidCGCrashes) {
+    if (mLastCGContext) {
+      ::CGContextRelease(mLastCGContext);
+    }
+    mLastCGContext = aCGContext;
+    ::CGContextRetain(mLastCGContext);
+  }
 
   mUpdateRect.SetEmpty();
 }
 
+- (void)dealloc
+{
+  if (mLastCGContext) {
+    ::CGContextRelease(mLastCGContext);
+  }
+  [super dealloc];
+}
+
 @end
 
-void* mozilla::plugins::PluginUtilsOSX::GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance) {
+void* mozilla::plugins::PluginUtilsOSX::GetCGLayer(DrawPluginFunc aFunc, void* aPluginInstance,
+                                                   bool aAvoidCGCrashes)
+{
   CGBridgeLayer *bridgeLayer = [[CGBridgeLayer alloc] init ];
-  [bridgeLayer setDrawFunc:aFunc pluginInstance:aPluginInstance];
+  [bridgeLayer setDrawFunc:aFunc
+            pluginInstance:aPluginInstance
+            avoidCGCrashes:aAvoidCGCrashes];
   return bridgeLayer;
 }
 
@@ -77,6 +153,7 @@ void mozilla::plugins::PluginUtilsOSX::Repaint(void *caLayer, nsIntRect aRect) {
   [bridgeLayer setNeedsDisplay];
   [bridgeLayer displayIfNeeded];
   [CATransaction commit];
+  [bridgeLayer protectLastCGContext];
 }
 
 @interface EventProcessor : NSObject {

widget/cocoa/nsCocoaFeatures.h

@@ -13,6 +13,7 @@ class nsCocoaFeatures {
 public:
   static int32_t OSXVersion();
   static bool OnLionOrLater();
+  static bool OnMountainLionOrLater();
   static bool SupportCoreAnimationPlugins();
 
 private:

widget/cocoa/nsCocoaFeatures.mm

@@ -8,6 +8,7 @@
 #define MAC_OS_X_VERSION_10_5_HEX 0x00001050
 #define MAC_OS_X_VERSION_10_6_HEX 0x00001060
 #define MAC_OS_X_VERSION_10_7_HEX 0x00001070
+#define MAC_OS_X_VERSION_10_8_HEX 0x00001080
 
 // This API will not work for OS X 10.10, see Gestalt.h.
 
@@ -54,3 +55,9 @@ nsCocoaFeatures::OnLionOrLater()
     return (OSXVersion() >= MAC_OS_X_VERSION_10_7_HEX);
 }
 
+/* static */ bool
+nsCocoaFeatures::OnMountainLionOrLater()
+{
+    return (OSXVersion() >= MAC_OS_X_VERSION_10_8_HEX);
+}
+