Bug 952051 - Change the GetOperator call in FlushGroup to take into account the mix-blend-mode operator for the group. r=roc

Create a new GetEffectiveOperator function that also takes into acount the mix-blend-mode operator for basic layers. Replace logic that chooses between the effective mix blend mode and the operator returned by GetOperator, with calls to this function.

gfx/2d/DrawTargetCairo.cpp

@@ -958,6 +958,8 @@ DrawTargetCairo::MaskSurface(const Pattern &aSource,
   cairo_matrix_init_translate (&matrix, -aOffset.x, -aOffset.y);
   cairo_pattern_set_matrix (mask, &matrix);
 
+  cairo_set_operator(mContext, GfxOpToCairoOp(aOptions.mCompositionOp));
+
   cairo_mask(mContext, mask);
 
   cairo_surface_destroy(surf);

gfx/layers/basic/BasicCanvasLayer.cpp

@@ -5,6 +5,7 @@
 
 #include "BasicCanvasLayer.h"
 #include "basic/BasicLayers.h"          // for BasicLayerManager
+#include "basic/BasicLayersImpl.h"      // for GetEffectiveOperator
 #include "mozilla/mozalloc.h"           // for operator new
 #include "nsAutoPtr.h"                  // for nsRefPtr
 #include "nsCOMPtr.h"                   // for already_AddRefed
@@ -29,11 +30,10 @@ BasicCanvasLayer::Paint(DrawTarget* aTarget, SourceSurface* aMaskSurface)
   UpdateTarget();
   FireDidTransactionCallback();
 
-  CompositionOp mixBlendMode = GetEffectiveMixBlendMode();
   PaintWithOpacity(aTarget,
                    GetEffectiveOpacity(),
                    aMaskSurface,
-                   mixBlendMode != CompositionOp::OP_OVER ? mixBlendMode : GetOperator());
+                   GetEffectiveOperator(this));
 }
 
 void

gfx/layers/basic/BasicColorLayer.cpp

@@ -48,9 +48,8 @@ public:
     if (IsHidden()) {
       return;
     }
-    CompositionOp mixBlendMode = GetEffectiveMixBlendMode();
-    CompositionOp op =
-      mixBlendMode != CompositionOp::OP_OVER ? mixBlendMode : GetOperator();
+
+    CompositionOp op = GetEffectiveOperator(this);
 
     DrawOptions opts = DrawOptions();
     opts.mCompositionOp = op;

gfx/layers/basic/BasicImageLayer.cpp

@@ -129,9 +129,7 @@ BasicImageLayer::GetAndPaintCurrentImage(DrawTarget* aTarget,
     // The visible region can extend outside the image, so just draw
     // within the image bounds.
     SurfacePattern pat(surf, ExtendMode::CLAMP, Matrix(), ToFilter(mFilter));
-    CompositionOp mixBlendMode = GetEffectiveMixBlendMode();
-    CompositionOp op =
-    mixBlendMode != CompositionOp::OP_OVER ? mixBlendMode : GetOperator();
+    CompositionOp op = GetEffectiveOperator(this);
     DrawOptions opts(aOpacity, op);
 
     aTarget->MaskSurface(pat, aMaskSurface, Point(0, 0), opts);
@@ -171,9 +169,7 @@ BasicImageLayer::DeprecatedGetAndPaintCurrentImage(gfxContext* aContext,
   // The visible region can extend outside the image, so just draw
   // within the image bounds.
   if (aContext) {
-    CompositionOp mixBlendMode = GetEffectiveMixBlendMode();
-    CompositionOp op =
-      mixBlendMode != CompositionOp::OP_OVER ? mixBlendMode : GetOperator();
+    CompositionOp op = GetEffectiveOperator(this);
     AutoSetOperator setOptimizedOperator(aContext, ThebesOp(op));
 
     DeprecatedPaintContext(pat,

gfx/layers/basic/BasicLayerManager.cpp

@@ -861,9 +861,10 @@ BasicLayerManager::FlushGroup(PaintLayerContext& aPaintContext, bool aNeedsClipT
       gfxUtils::ClipToRegion(aPaintContext.mTarget,
                              aPaintContext.mLayer->GetEffectiveVisibleRegion());
     }
-    BasicContainerLayer* container = static_cast<BasicContainerLayer*>(aPaintContext.mLayer);
-    AutoSetOperator setOperator(aPaintContext.mTarget,
-                                ThebesOp(container->GetOperator()));
+
+    CompositionOp op = GetEffectiveOperator(aPaintContext.mLayer);
+    AutoSetOperator setOperator(aPaintContext.mTarget, ThebesOp(op));
+
     PaintWithMask(aPaintContext.mTarget, aPaintContext.mLayer->GetEffectiveOpacity(),
                   aPaintContext.mLayer->GetMaskLayer());
   }

gfx/layers/basic/BasicLayersImpl.cpp

@@ -135,6 +135,18 @@ ToData(Layer* aLayer)
   return static_cast<BasicImplData*>(aLayer->ImplData());
 }
 
+gfx::CompositionOp
+GetEffectiveOperator(Layer* aLayer)
+{
+  CompositionOp op = aLayer->GetEffectiveMixBlendMode();
+
+  if (op != CompositionOp::OP_OVER) {
+    return op;
+  }
+
+  return ToData(aLayer)->GetOperator();
+}
+
 ShadowableLayer*
 ToShadowable(Layer* aLayer)
 {

gfx/layers/basic/BasicLayersImpl.h

@@ -94,6 +94,21 @@ FillWithMask(gfxContext* aContext, float aOpacity, Layer* aMaskLayer);
 BasicImplData*
 ToData(Layer* aLayer);
 
+/**
+ * Returns the operator to be used when blending and compositing this layer.
+ * Currently there is no way to specify both a blending and a compositing
+ * operator other than normal and source over respectively.
+ *
+ * If the layer has
+ * an effective blend mode operator other than normal, as returned by
+ * GetEffectiveMixBlendMode, this operator is used for blending, and source
+ * over is used for compositing.
+ * If the blend mode for this layer is normal, the compositing operator
+ * returned by GetOperator is used.
+ */
+gfx::CompositionOp
+GetEffectiveOperator(Layer* aLayer);
+
 }
 }
 

gfx/layers/basic/BasicThebesLayer.cpp

@@ -62,7 +62,7 @@ BasicThebesLayer::PaintThebes(gfxContext* aContext,
   }
 
   float opacity = GetEffectiveOpacity();
-  CompositionOp mixBlendMode = GetEffectiveMixBlendMode();
+  CompositionOp effectiveOperator = GetEffectiveOperator(this);
 
   if (!BasicManager()->IsRetained()) {
     NS_ASSERTION(readbackUpdates.IsEmpty(), "Can't do readback for non-retained layer");
@@ -84,16 +84,14 @@ BasicThebesLayer::PaintThebes(gfxContext* aContext,
 
       bool needsClipToVisibleRegion = GetClipToVisibleRegion();
       bool needsGroup = opacity != 1.0 ||
-                        GetOperator() != CompositionOp::OP_OVER ||
-                        mixBlendMode != CompositionOp::OP_OVER ||
+                        effectiveOperator != CompositionOp::OP_OVER ||
                         aMaskLayer;
       nsRefPtr<gfxContext> groupContext;
       if (needsGroup) {
         groupContext =
           BasicManager()->PushGroupForLayer(aContext, this, toDraw,
                                             &needsClipToVisibleRegion);
-        if (GetOperator() != CompositionOp::OP_OVER ||
-            mixBlendMode != CompositionOp::OP_OVER) {
+        if (effectiveOperator != CompositionOp::OP_OVER) {
           needsClipToVisibleRegion = true;
         }
       } else {
@@ -106,9 +104,7 @@ BasicThebesLayer::PaintThebes(gfxContext* aContext,
         if (needsClipToVisibleRegion) {
           gfxUtils::ClipToRegion(aContext, toDraw);
         }
-        CompositionOp op =
-          mixBlendMode != CompositionOp::OP_OVER ? mixBlendMode : GetOperator();
-        AutoSetOperator setOptimizedOperator(aContext, ThebesOp(op));
+        AutoSetOperator setOptimizedOperator(aContext, ThebesOp(effectiveOperator));
         PaintWithMask(aContext, opacity, aMaskLayer);
       }