Bug 1023336, part 1 - Add a Moz2D helper to consistently limit the size of data-wrapping DataSourceSurface buffers. r=Bas

gfx/2d/DataSurfaceHelpers.cpp

@@ -7,6 +7,8 @@
 
 #include "2D.h"
 #include "DataSurfaceHelpers.h"
+#include "Logging.h"
+#include "mozilla/MathAlgorithms.h"
 #include "Tools.h"
 
 namespace mozilla {
@@ -167,5 +169,30 @@ ClearDataSourceSurface(DataSourceSurface *aSurface)
   aSurface->Unmap();
 }
 
+size_t
+BufferSizeFromStrideAndHeight(int32_t aStride,
+                              int32_t aHeight,
+                              int32_t aExtraBytes)
+{
+  if (MOZ_UNLIKELY(aHeight <= 0)) {
+    return 0;
+  }
+
+  // We limit the length returned to values that can be represented by int32_t
+  // because we don't want to allocate buffers any bigger than that. This
+  // allows for a buffer size of over 2 GiB which is already rediculously
+  // large and will make the process janky. (Note the choice of the signed type
+  // is deliberate because we specifically don't want the returned value to
+  // overflow if someone stores the buffer length in an int32_t variable.)
+
+  CheckedInt32 requiredBytes =
+    CheckedInt32(aStride) * CheckedInt32(aHeight) + CheckedInt32(aExtraBytes);
+  if (MOZ_UNLIKELY(!requiredBytes.isValid())) {
+    gfxWarning() << "Buffer size too big; returning zero";
+    return 0;
+  }
+  return requiredBytes.value();
+}
+
 }
 }

gfx/2d/DataSurfaceHelpers.h

@@ -52,6 +52,22 @@ SurfaceToPackedBGR(DataSourceSurface *aSurface);
 void
 ClearDataSourceSurface(DataSourceSurface *aSurface);
 
+/**
+ * Multiplies aStride and aHeight and makes sure the result is limited to
+ * something sane. To keep things consistent, this should always be used
+ * wherever we allocate a buffer based on surface stride and height.
+ *
+ * @param aExtra Optional argument to specify an additional number of trailing
+ *   bytes (useful for creating intermediate surfaces for filters, for
+ *   example).
+ *
+ * @return The result of the multiplication if it is acceptable, or else zero.
+ */
+size_t
+BufferSizeFromStrideAndHeight(int32_t aStride,
+                              int32_t aHeight,
+                              int32_t aExtraBytes = 0);
+
 }
 }